diff --git a/.env.example b/.env.example index 2aa11780748..5f0285650a9 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,7 @@ SOCKET_PORT='3001' # AI MODELS AI_EMBEDDING_MODELS='[{"model": "text-embeddings-inference:llmrails/ember-v1", "url": "http://localhost:3040/"}]' AI_GENERATION_MODELS='[{"model": "text-generation-inference:TheBloke/zephyr-7b-beta", "url": "http://localhost:3050/"}]' -AI_EMBEDDER_ENABLED='true' +AI_EMBEDDER_WORKERS='1' # APPLICATION # AMPLITUDE_WRITE_KEY='key_AMPLITUDE_WRITE_KEY' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 709f0a2d818..4da61b03eac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: id-token: "write" services: postgres: - image: pgvector/pgvector:pg15 + image: pgvector/pgvector:0.6.2-pg15 # This env variables must be the same in the file PARABOL_BUILD_ENV_PATH env: POSTGRES_PASSWORD: "temppassword" @@ -143,6 +143,6 @@ jobs: uses: ravsamhq/notify-slack-action@v2 with: status: ${{ job.status }} - notify_when: 'failure' + notify_when: "failure" env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GH_ACTIONS_NOTIFICATIONS }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8378696b622..171e080964f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: id-token: "write" services: postgres: - image: postgres:15.4 + image: pgvector/pgvector:0.6.2-pg15 # This env variables must be the same in the file PARABOL_BUILD_ENV_PATH env: POSTGRES_PASSWORD: "temppassword" @@ -78,7 +78,6 @@ jobs: yarn db:migrate yarn pg:migrate up yarn pg:build - yarn pg:generate - name: Build for testing run: yarn build @@ -86,9 +85,6 @@ jobs: - name: Verify source is clean run: git diff --quiet HEAD || (echo "Changes in generated files detected"; git diff; exit 1) - - name: Check Code Quality - run: yarn codecheck - - name: Run Predeploy for Testing run: yarn predeploy @@ -100,6 +96,12 @@ jobs: wait-on: | http://localhost:3000/graphql + - name: Kysely Codegen + run: yarn pg:generate + + - name: Check Code Quality + run: yarn codecheck + - name: Run server tests run: yarn test:server -- --reporters=default --reporters=jest-junit env: @@ -139,6 +141,6 @@ jobs: uses: ravsamhq/notify-slack-action@v2 with: status: ${{ job.status }} - notify_when: 'failure' + notify_when: "failure" env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GH_ACTIONS_NOTIFICATIONS }} diff --git a/.release-please-manifest.json b/.release-please-manifest.json index bae38c98366..2fb3566ccf3 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "7.23.1" + ".": "7.24.1" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 294dbfc8830..b5ed33f4256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ This project adheres to [Semantic Versioning](http://semver.org/). This CHANGELOG follows conventions [outlined here](http://keepachangelog.com/). +## [7.24.1](https://github.com/ParabolInc/parabol/compare/v7.24.0...v7.24.1) (2024-04-02) + + +### Fixed + +* embedder doesn't dive deep into schema ([#9582](https://github.com/ParabolInc/parabol/issues/9582)) ([8cdd901](https://github.com/ParabolInc/parabol/commit/8cdd9014c3277905605c6544de92d9ac2833a6e9)) +* embedder errors in embed length ([#9584](https://github.com/ParabolInc/parabol/issues/9584)) ([341b4b7](https://github.com/ParabolInc/parabol/commit/341b4b797ec6444066244f25916803e64c03258c)) +* Fetch CORS resources from network ([#9586](https://github.com/ParabolInc/parabol/issues/9586)) ([b6ddfa5](https://github.com/ParabolInc/parabol/commit/b6ddfa5755394633e83dadd0178234ef740454ea)) + +## [7.24.0](https://github.com/ParabolInc/parabol/compare/v7.23.1...v7.24.0) (2024-03-29) + + +### Added + +* prepare embedder for Production ([#9517](https://github.com/ParabolInc/parabol/issues/9517)) ([538c95c](https://github.com/ParabolInc/parabol/commit/538c95ce4dc7d4839b3e813006cb20e1b7d1d1c8)) + + +### Changed + +* fix tsconfig problems ([#9579](https://github.com/ParabolInc/parabol/issues/9579)) ([d1af0f1](https://github.com/ParabolInc/parabol/commit/d1af0f164c629e8fc075278cd63475e8913f4295)) + ## [7.23.1](https://github.com/ParabolInc/parabol/compare/v7.23.0...v7.23.1) (2024-03-28) diff --git a/docker/images/parabol-ubi/README.md b/docker/images/parabol-ubi/README.md index 920c6d48fd8..518269140bc 100644 --- a/docker/images/parabol-ubi/README.md +++ b/docker/images/parabol-ubi/README.md @@ -16,21 +16,21 @@ Recommended: ## Variables -| Name | Description | Possible values | Recommended value | -| -------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | ------------------------------------------------------------------- | -| `postgresql_tag` | PostgreSQL version from the [Docker image](https://hub.docker.com/_/postgres) | `Any tag` | `15.4` | -| `rethinkdb_tag` | RethinkDB version from the [Docker image](https://hub.docker.com/_/rethinkdb) | `Any tag` | `2.4.2` | -| `redis_tag` | Redis version from the [Docker image](https://hub.docker.com/_/redis) | `Any tag` | `7.0-alpine` | -| `_BUILD_ENV_PATH` | File `.env` used by the application during the build process | `Relative path from the root level of the repository` | `docker/parabol-ubi/environments/basic-env` | -| `_NODE_VERSION` | Node version, used by Docker to use the Docker image node:\_NODE_VERSION as base image to build | `Same as in root package.json` | | -| `_DOCKERFILE` | Dockerfile used to build the image | `Relative path from the root level of the repository` | `./docker/parabol-ubi/dockerfiles/basic.dockerfile` | -| `_DOCKER_REPOSITORY` | The destination repository | `String` | `parabol` | -| `_DOCKER_TAG` | Tag for the produced image | `String` | | +| Name | Description | Possible values | Recommended value | +| -------------------- | ----------------------------------------------------------------------------------------------- | ----------------------------------------------------- | --------------------------------------------------- | +| `postgresql_tag` | PostgreSQL version from the [Docker image](https://hub.docker.com/r/pgvector/pgvector) | `Any tag` | `0.6.2-pg15` | +| `rethinkdb_tag` | RethinkDB version from the [Docker image](https://hub.docker.com/_/rethinkdb) | `Any tag` | `2.4.2` | +| `redis_tag` | Redis version from the [Docker image](https://hub.docker.com/_/redis) | `Any tag` | `7.0-alpine` | +| `_BUILD_ENV_PATH` | File `.env` used by the application during the build process | `Relative path from the root level of the repository` | `docker/parabol-ubi/environments/basic-env` | +| `_NODE_VERSION` | Node version, used by Docker to use the Docker image node:\_NODE_VERSION as base image to build | `Same as in root package.json` | | +| `_DOCKERFILE` | Dockerfile used to build the image | `Relative path from the root level of the repository` | `./docker/parabol-ubi/dockerfiles/basic.dockerfile` | +| `_DOCKER_REPOSITORY` | The destination repository | `String` | `parabol` | +| `_DOCKER_TAG` | Tag for the produced image | `String` | | Example of variables: ```commandLine -export postgresql_tag=15.4; \ +export postgresql_tag=0.6.2-pg15; \ export rethinkdb_tag=2.4.2; \ export redis_tag=7.0-alpine; \ export _BUILD_ENV_PATH=docker/parabol-ubi/environments/basic-env; \ @@ -61,7 +61,7 @@ cp $_BUILD_ENV_PATH ./.env > :warning: Stop all database containers you might have running before executing the following command. If other database containers are running, some ports might be already taken. ```commandLine -docker run --name temp-postgres -e POSTGRES_PASSWORD=temppassword -e POSTGRES_USER=tempuser -e POSTGRES_DB=tempdb -d -p 5432:5432 postgres:$postgresql_tag && \ +docker run --name temp-postgres -e POSTGRES_PASSWORD=temppassword -e POSTGRES_USER=tempuser -e POSTGRES_DB=tempdb -d -p 5432:5432 pgvector/pgvector:$postgresql_tag && \ docker run --name temp-rethinkdb -d -p 28015:28015 -p 29015:29015 -p 8080:8080 rethinkdb:$rethinkdb_tag && \ docker run --name temp-redis -d -p 6379:6379 redis:$redis_tag ``` diff --git a/docker/images/parabol-ubi/environments/pipeline b/docker/images/parabol-ubi/environments/pipeline index cfc707c746b..cd111fab88b 100644 --- a/docker/images/parabol-ubi/environments/pipeline +++ b/docker/images/parabol-ubi/environments/pipeline @@ -54,3 +54,7 @@ STRIPE_PUBLISHABLE_KEY='pk_test_MNoKbCzQX0lhktuxxI7M14wd' STRIPE_SECRET_KEY='' STRIPE_WEBHOOK_SECRET='' HUBSPOT_API_KEY='' +AI_EMBEDDING_MODELS='[{"model": "text-embeddings-inference:llmrails/ember-v1", "url": "http://localhost:3040/"}]' +AI_GENERATION_MODELS='[{"model": "text-generation-inference:TheBloke/zephyr-7b-beta", "url": "http://localhost:3050/"}]' +AI_EMBEDDER_WORKERS='1' +POSTGRES_USE_PGVECTOR='true' diff --git a/docker/stacks/development/docker-compose.yml b/docker/stacks/development/docker-compose.yml index 91adbbb6578..a4509d051b9 100644 --- a/docker/stacks/development/docker-compose.yml +++ b/docker/stacks/development/docker-compose.yml @@ -70,7 +70,7 @@ services: networks: parabol-network: text-embeddings-inference: - image: ghcr.io/huggingface/text-embeddings-inference:cpu-0.6 + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 command: - "--model-id=llmrails/ember-v1" platform: linux/x86_64 diff --git a/docker/stacks/single-tenant-host/docker-compose.yaml b/docker/stacks/single-tenant-host/docker-compose.yml similarity index 98% rename from docker/stacks/single-tenant-host/docker-compose.yaml rename to docker/stacks/single-tenant-host/docker-compose.yml index e5f662ed74a..a336201bfe3 100644 --- a/docker/stacks/single-tenant-host/docker-compose.yaml +++ b/docker/stacks/single-tenant-host/docker-compose.yml @@ -17,7 +17,7 @@ services: postgres: container_name: postgres profiles: ["databases"] - image: postgres:15.4 + image: pgvector/pgvector:0.6.2-pg15 restart: always env_file: .env environment: diff --git a/package.json b/package.json index 82179da713f..a6b2e22b178 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "description": "An open-source app for building smarter, more agile teams.", "author": "Parabol Inc. (http://github.com/ParabolInc)", "license": "AGPL-3.0", - "version": "7.23.1", + "version": "7.24.1", "repository": { "type": "git", "url": "https://github.com/ParabolInc/parabol" @@ -92,18 +92,19 @@ "@types/dotenv": "^6.1.1", "@types/jscodeshift": "^0.11.3", "@types/lodash.toarray": "^4.4.7", - "@typescript-eslint/eslint-plugin": "^6.21.0", - "@typescript-eslint/parser": "^6.21.0", + "@typescript-eslint/eslint-plugin": "^7.4.0", + "@typescript-eslint/parser": "^7.4.0", "autoprefixer": "^10.4.13", "babel-loader": "^9.1.2", "concurrently": "^8.0.1", "copy-webpack-plugin": "^11.0.0", - "eslint-config-prettier": "^8.5.0", + "eslint": "^8.57.0", + "eslint-config-prettier": "^9.1.0", "graphql": "15.7.2", "html-webpack-plugin": "^5.5.0", "husky": "^7.0.4", "jscodeshift": "^0.14.0", - "kysely": "^0.27.2", + "kysely": "^0.27.3", "kysely-codegen": "^0.11.0", "lerna": "^6.4.1", "mini-css-extract-plugin": "^2.7.2", diff --git a/packages/chronos/package.json b/packages/chronos/package.json index b2e5061c6bc..b8e0b4886a7 100644 --- a/packages/chronos/package.json +++ b/packages/chronos/package.json @@ -1,6 +1,6 @@ { "name": "chronos", - "version": "7.23.1", + "version": "7.24.1", "description": "A cron job scheduler", "author": "Matt Krick ", "homepage": "https://github.com/ParabolInc/parabol/tree/master/packages/chronos#readme", @@ -25,6 +25,6 @@ }, "dependencies": { "cron": "^2.3.1", - "parabol-server": "7.23.1" + "parabol-server": "7.24.1" } } diff --git a/packages/chronos/tsconfig.json b/packages/chronos/tsconfig.json index 00f66831a3b..87e7ef08687 100644 --- a/packages/chronos/tsconfig.json +++ b/packages/chronos/tsconfig.json @@ -9,13 +9,6 @@ "~/*": ["client/*"] }, "outDir": "lib", - "lib": ["esnext"], - "types": ["node"] - }, - "files": [ - "chronos.ts", - "../server/types/webpackEnv.ts", - "../server/types/modules.d.ts", - "../client/modules/email/components/SummaryEmail/MeetingSummaryEmail/MeetingSummaryEmail.tsx" - ] + "lib": ["esnext"] + } } diff --git a/packages/client/components/AnimatedFade.tsx b/packages/client/components/AnimatedFade.tsx deleted file mode 100644 index f1f612ec4ff..00000000000 --- a/packages/client/components/AnimatedFade.tsx +++ /dev/null @@ -1,74 +0,0 @@ -/* Deprecated. See internals of useMenuPortal */ - -import {ClassNames} from '@emotion/core' -import React, {Component, ReactNode} from 'react' -import {CSSTransition} from 'react-transition-group' - -interface Props extends CSSTransition { - appear?: boolean - in?: boolean - onExited?: () => void - exit?: boolean - unmountOnExit?: boolean - children: ReactNode - duration?: number - slide?: number -} - -// eslint-disable-next-line -class AnimatedFade extends Component { - render() { - const {children, duration = 100, slide = 32, ...props} = this.props - - const classNames = (css) => { - const enter = css({ - opacity: 0, - transform: `translate3d(0, ${slide}px, 0)` - }) - const enterActive = css({ - opacity: '1 !important' as any, - transform: 'translate3d(0, 0, 0) !important', - transition: `all ${duration}ms ease-in !important` - }) - - const exit = css({ - opacity: 1, - transform: 'translate3d(0, 0, 0)' - }) - - const exitActive = css({ - opacity: '0 !important' as any, - transform: `translate3d(0, ${-slide}px, 0) !important`, - transition: `all ${duration}ms ease-in !important` - }) - return { - appear: enter, - appearActive: enterActive, - enter, - enterActive, - exit, - exitActive - } - } - return ( - - {({css}) => { - return ( - - {children} - - ) - }} - - ) - } -} - -export default AnimatedFade diff --git a/packages/client/components/AzureDevOpsScopingSelectAllIssues.tsx b/packages/client/components/AzureDevOpsScopingSelectAllIssues.tsx index d4f80d54fe0..24c42792294 100644 --- a/packages/client/components/AzureDevOpsScopingSelectAllIssues.tsx +++ b/packages/client/components/AzureDevOpsScopingSelectAllIssues.tsx @@ -3,6 +3,7 @@ import graphql from 'babel-plugin-relay/macro' import React from 'react' import {useFragment} from 'react-relay' import useUnusedRecords from '~/hooks/useUnusedRecords' +import {AzureDevOpsScopingSelectAllIssues_workItems$key} from '../__generated__/AzureDevOpsScopingSelectAllIssues_workItems.graphql' import useAtmosphere from '../hooks/useAtmosphere' import useMutationProps from '../hooks/useMutationProps' import UpdatePokerScopeMutation from '../mutations/UpdatePokerScopeMutation' @@ -11,7 +12,6 @@ import {PALETTE} from '../styles/paletteV3' import {Threshold} from '../types/constEnums' import AzureDevOpsClientManager from '../utils/AzureDevOpsClientManager' import getSelectAllTitle from '../utils/getSelectAllTitle' -import {AzureDevOpsScopingSelectAllIssues_workItems$key} from '../__generated__/AzureDevOpsScopingSelectAllIssues_workItems.graphql' import Checkbox from './Checkbox' const Item = styled('div')({ @@ -42,7 +42,7 @@ interface Props { } const AzureDevOpsScopingSelectAllIssues = (props: Props) => { - const {meetingId, usedServiceTaskIds, workItems: workItemsRef, providerId} = props + const {meetingId, usedServiceTaskIds, workItems: workItemsRef} = props const workItems = useFragment( graphql` fragment AzureDevOpsScopingSelectAllIssues_workItems on AzureDevOpsWorkItemEdge @@ -57,7 +57,7 @@ const AzureDevOpsScopingSelectAllIssues = (props: Props) => { workItemsRef ) const atmosphere = useAtmosphere() - const {onCompleted, onError, submitMutation, submitting, error} = useMutationProps() + const {onCompleted, onError, submitMutation, error} = useMutationProps() const getProjectId = (url: URL) => { const firstIndex = url.pathname.indexOf('/', 1) const seconedIndex = url.pathname.indexOf('/', firstIndex + 1) diff --git a/packages/client/hooks/useAtlassianSites.ts b/packages/client/hooks/useAtlassianSites.ts deleted file mode 100644 index 8fdb3a66567..00000000000 --- a/packages/client/hooks/useAtlassianSites.ts +++ /dev/null @@ -1,43 +0,0 @@ -import {useEffect, useRef, useState} from 'react' -import {Unpromise} from '../types/generics' -import AtlassianClientManager from '../utils/AtlassianClientManager' -import {AccessibleResource} from '../utils/AtlassianManager' - -const useAtlassianSites = (accessToken?: string) => { - const isMountedRef = useRef(true) - const [sites, setSites] = useState([]) - const [status, setStatus] = useState(null) - useEffect(() => { - const manager = new AtlassianClientManager(accessToken || '') - const fetchSites = async () => { - let res: Unpromise> - try { - res = await manager.getAccessibleResources() - } catch (e) { - if (isMountedRef.current) { - setStatus('error') - } - return - } - if (isMountedRef.current) { - if (Array.isArray(res)) { - setStatus('loaded') - setSites(res) - } else { - setStatus('error') - } - } - } - - if (accessToken && isMountedRef.current) { - setStatus('loading') - fetchSites().catch() - } - return () => { - isMountedRef.current = false - } - }, [accessToken]) - return {sites, status} -} - -export default useAtlassianSites diff --git a/packages/client/modules/email/components/MeetingSummaryEmailRootSSR.tsx b/packages/client/modules/email/components/MeetingSummaryEmailRootSSR.tsx index 9ad3310cd7d..bd17389a344 100644 --- a/packages/client/modules/email/components/MeetingSummaryEmailRootSSR.tsx +++ b/packages/client/modules/email/components/MeetingSummaryEmailRootSSR.tsx @@ -2,7 +2,6 @@ import graphql from 'babel-plugin-relay/macro' import {MeetingSummaryEmailRootSSRQuery} from 'parabol-client/__generated__/MeetingSummaryEmailRootSSRQuery.graphql' import React from 'react' import {useLazyLoadQuery} from 'react-relay' -import {Environment} from 'relay-runtime' import {EMAIL_CORS_OPTIONS} from '../../../types/cors' import makeAppURL from '../../../utils/makeAppURL' import MeetingSummaryEmail from './SummaryEmail/MeetingSummaryEmail/MeetingSummaryEmail' diff --git a/packages/client/modules/email/components/NotificationSummaryEmailRoot.tsx b/packages/client/modules/email/components/NotificationSummaryEmailRoot.tsx index bcf16c49bb0..cd610f6758d 100644 --- a/packages/client/modules/email/components/NotificationSummaryEmailRoot.tsx +++ b/packages/client/modules/email/components/NotificationSummaryEmailRoot.tsx @@ -56,7 +56,7 @@ const NotificationSummaryEmailRoot = (props: NotificationSummaryRootProps) => { (edge) => edge.node.status === 'UNREAD' && new Date(edge.node.createdAt) > new Date(Date.now() - ms('1d')) && - NOTIFICATION_TEMPLATE_TYPE[edge.node.type] // Filter down to the notifications that have been implemented. + NOTIFICATION_TEMPLATE_TYPE[edge.node.type as keyof typeof NOTIFICATION_TEMPLATE_TYPE] // Filter down to the notifications that have been implemented. ) .map((edge) => edge.node) .slice(0, MAX_EMAIL_NOTIFICATIONS) diff --git a/packages/client/modules/email/components/SummaryEmail/MeetingSummaryEmail/MeetingSummaryEmail.tsx b/packages/client/modules/email/components/SummaryEmail/MeetingSummaryEmail/MeetingSummaryEmail.tsx index 5a2611456f8..b49644af0f5 100644 --- a/packages/client/modules/email/components/SummaryEmail/MeetingSummaryEmail/MeetingSummaryEmail.tsx +++ b/packages/client/modules/email/components/SummaryEmail/MeetingSummaryEmail/MeetingSummaryEmail.tsx @@ -36,20 +36,6 @@ const pagePadding = { paddingTop: 24 } -declare module 'react' { - interface TdHTMLAttributes { - height?: string | number - width?: string | number - bgcolor?: string - } - interface TableHTMLAttributes { - align?: 'center' | 'left' | 'right' - bgcolor?: string - height?: string | number - width?: string | number - } -} - const PagePadding = () => { return ( diff --git a/packages/client/mutations/handlers/handleUpdateTeamMembers.ts b/packages/client/mutations/handlers/handleUpdateTeamMembers.ts index f62a323f291..6ccecccea3c 100644 --- a/packages/client/mutations/handlers/handleUpdateTeamMembers.ts +++ b/packages/client/mutations/handlers/handleUpdateTeamMembers.ts @@ -1,8 +1,12 @@ +import {RecordProxy, RecordSourceSelectorProxy} from 'relay-runtime' import fromTeamMemberId from '../../utils/relay/fromTeamMemberId' import safeRemoveNodeFromArray from '../../utils/relay/safeRemoveNodeFromArray' import pluralizeHandler from './pluralizeHandler' -const handleUpdateTeamMember = (updatedTeamMember, store) => { +const handleUpdateTeamMember = ( + updatedTeamMember: RecordProxy<{id: string}>, + store: RecordSourceSelectorProxy +) => { if (!updatedTeamMember) return const {teamId} = fromTeamMemberId(updatedTeamMember.getValue('id')) const isNotRemoved = updatedTeamMember.getValue('isNotRemoved') @@ -11,7 +15,7 @@ const handleUpdateTeamMember = (updatedTeamMember, store) => { const sorts = ['preferredName'] if (isNotRemoved) { sorts.forEach((sortBy) => { - const teamMembers = team.getLinkedRecords('teamMembers', {sortBy}) + const teamMembers = team.getLinkedRecords<[]>('teamMembers', {sortBy}) if (!teamMembers) return teamMembers.sort((a, b) => (a.getValue(sortBy) > b.getValue(sortBy) ? 1 : -1)) team.setLinkedRecords(teamMembers, 'teamMembers', {sortBy}) @@ -19,8 +23,7 @@ const handleUpdateTeamMember = (updatedTeamMember, store) => { } else { const teamMemberId = updatedTeamMember.getValue('id') sorts.forEach((sortBy) => { - const teamMembers = team.getLinkedRecords('teamMembers', {sortBy}) - safeRemoveNodeFromArray(teamMemberId, teamMembers, 'teamMembers', { + safeRemoveNodeFromArray(teamMemberId, team, 'teamMembers', { storageKeyArgs: {sortBy} }) }) diff --git a/packages/client/package.json b/packages/client/package.json index 3f105e730fb..51b9eb2aead 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -3,7 +3,7 @@ "description": "An open-source app for building smarter, more agile teams.", "author": "Parabol Inc. (http://github.com/ParabolInc)", "license": "AGPL-3.0", - "version": "7.23.1", + "version": "7.24.1", "repository": { "type": "git", "url": "https://github.com/ParabolInc/parabol" @@ -31,7 +31,7 @@ "@types/draft-js": "^0.10.24", "@types/fbjs": "^3.0.4", "@types/humanize-duration": "3.27.1", - "@types/jest": "^29.5.1", + "@types/jest": "^29.5.12", "@types/json2csv": "^4.4.0", "@types/jwt-decode": "^2.1.0", "@types/linkify-it": "^3.0.2", @@ -50,8 +50,6 @@ "@types/stripe-v2": "^2.0.1", "babel-plugin-relay": "^12.0.0", "debug": "^4.1.1", - "eslint": "^8.2.0", - "eslint-config-prettier": "^8.5.0", "eslint-plugin-emotion": "^10.0.14", "eslint-plugin-react": "^7.16.0", "eslint-plugin-react-hooks": "^1.6.1", @@ -75,6 +73,8 @@ "@mui/icons-material": "^5.8.4", "@mui/material": "^5.9.2", "@mui/x-date-pickers": "^6.3.1", + "@radix-ui/react-alert-dialog": "1.0.5", + "@radix-ui/react-avatar": "^1.0.4", "@radix-ui/react-collapsible": "^1.0.3", "@radix-ui/react-dialog": "^1.0.4", "@radix-ui/react-dropdown-menu": "^2.0.4", @@ -82,8 +82,6 @@ "@radix-ui/react-scroll-area": "^1.0.3", "@radix-ui/react-select": "^1.2.2", "@radix-ui/react-slot": "^1.0.2", - "@radix-ui/react-avatar": "^1.0.4", - "@radix-ui/react-alert-dialog": "1.0.5", "@radix-ui/react-tooltip": "^1.0.7", "@sentry/browser": "^5.8.0", "@stripe/react-stripe-js": "^1.16.5", diff --git a/packages/client/serviceWorker/sw.ts b/packages/client/serviceWorker/sw.ts index e35d7f86db7..eb19da02807 100644 --- a/packages/client/serviceWorker/sw.ts +++ b/packages/client/serviceWorker/sw.ts @@ -79,8 +79,10 @@ const onFetch = async (event: FetchEvent) => { // request.mode could be 'no-cors' // By fetching the URL without specifying the mode the response will not be opaque const isParabolHosted = url.startsWith(PUBLIC_PATH) || url.startsWith(self.origin) - const req = isParabolHosted ? request.url : request - const networkRes = await fetch(req) + // if one of our assets is not in the service worker cache, then it's either fetched via network or served from the broswer cache. + // The browser cache most likely has incorrect CORS headers set, so we better always fetch from the network. + const req = isParabolHosted ? fetch(request.url, {cache: 'no-store'}) : fetch(request) + const networkRes = await req const cache = await caches.open(DYNAMIC_CACHE) // cloning here because I'm not sure if we must clone before reading the body cache.put(request.url, networkRes.clone()).catch(console.error) diff --git a/packages/client/serviceWorker/tsconfig.json b/packages/client/serviceWorker/tsconfig.json index ec0c5fb0b28..f393dcadb5f 100644 --- a/packages/client/serviceWorker/tsconfig.json +++ b/packages/client/serviceWorker/tsconfig.json @@ -1,7 +1,6 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { -// "composite": true, "lib": ["esnext", "webworker"], "target": "esnext", "module": "commonjs", @@ -11,6 +10,5 @@ "*" // resolve all absolute imports as imports relative to the client package ] } - }, -// "references": [{"path": "../"}] + } } diff --git a/packages/client/shared/gqlIds/EmbedderChannelId.ts b/packages/client/shared/gqlIds/EmbedderChannelId.ts new file mode 100644 index 00000000000..f1fb49341ef --- /dev/null +++ b/packages/client/shared/gqlIds/EmbedderChannelId.ts @@ -0,0 +1,9 @@ +export const EmbedderChannelId = { + join: (serverId: string) => `embedder:${serverId}`, + split: (id: string) => { + const [, serverId] = id.split(':') + return serverId + } +} + +export default EmbedderChannelId diff --git a/packages/client/tsconfig.json b/packages/client/tsconfig.json index 58280d064ea..32bb2473e60 100644 --- a/packages/client/tsconfig.json +++ b/packages/client/tsconfig.json @@ -3,15 +3,10 @@ "compilerOptions": { "baseUrl": ".", "paths": { - "~/*": ["*"], - "static/*": ["../../static/*"] + "~/*": ["*"] }, "outDir": "lib", - "lib": ["esnext", "dom"], - "types": ["node"] + "lib": ["esnext", "dom"] }, - "files": [ - "client.tsx", - ], "exclude": ["serviceWorker", "**/node_modules"] } diff --git a/packages/client/types/generics.ts b/packages/client/types/generics.ts index 663bbd7ac52..7a1d76dafb0 100644 --- a/packages/client/types/generics.ts +++ b/packages/client/types/generics.ts @@ -100,6 +100,9 @@ export type WithFieldsAsType = { : TObj[K] } +export type Tuple = R['length'] extends N ? R : Tuple +export type ParseInt = T extends `${infer Digit extends number}` ? Digit : never + declare global { interface Array { findLastIndex(predicate: (value: T, index: number, obj: T[]) => unknown, thisArg?: any): number diff --git a/packages/client/types/modules.d.ts b/packages/client/types/modules.d.ts index 92293d0cbbd..28b06e27aa7 100644 --- a/packages/client/types/modules.d.ts +++ b/packages/client/types/modules.d.ts @@ -20,6 +20,7 @@ declare module 'emoji-mart/dist-modern/utils/data.js' declare module 'emoji-mart/dist-modern/components/picker/nimble-picker' declare module 'react-textarea-autosize' declare module 'react-copy-to-clipboard' +declare module 'tayden-clusterfck' declare let __webpack_public_path__: string declare const __PRODUCTION__: string diff --git a/packages/client/types/reactHTML4.d.ts b/packages/client/types/reactHTML4.d.ts new file mode 100644 index 00000000000..995d555df4a --- /dev/null +++ b/packages/client/types/reactHTML4.d.ts @@ -0,0 +1,15 @@ +import 'react' + +declare module 'react' { + export interface TdHTMLAttributes { + height?: string | number + width?: string | number + bgcolor?: string + } + export interface TableHTMLAttributes { + align?: 'center' | 'left' | 'right' + bgcolor?: string + height?: string | number + width?: string | number + } +} diff --git a/packages/client/ui/AlertDialog/AlertDialogContent.tsx b/packages/client/ui/AlertDialog/AlertDialogContent.tsx index dbcd0de0582..12d5a79355c 100644 --- a/packages/client/ui/AlertDialog/AlertDialogContent.tsx +++ b/packages/client/ui/AlertDialog/AlertDialogContent.tsx @@ -1,8 +1,8 @@ -import * as React from 'react' import * as AlertDialogPrimitive from '@radix-ui/react-alert-dialog' import clsx from 'clsx' +import * as React from 'react' import {AlertDialogOverlay} from './AlertDialogOverlay' -import {AlertDialogPortal} from './AlertDialog' +import {AlertDialogPortal} from './AlertDialogPortal' const AlertDialogContent = React.forwardRef< HTMLDivElement, diff --git a/packages/client/ui/AlertDialog/AlertDialogPortal.tsx b/packages/client/ui/AlertDialog/AlertDialogPortal.tsx index c4e8d56916f..f572b2d9baa 100644 --- a/packages/client/ui/AlertDialog/AlertDialogPortal.tsx +++ b/packages/client/ui/AlertDialog/AlertDialogPortal.tsx @@ -1,3 +1,3 @@ import * as AlertDialogPrimitive from '@radix-ui/react-alert-dialog' -const AlertDialogPortal = AlertDialogPrimitive.Portal +export const AlertDialogPortal = AlertDialogPrimitive.Portal diff --git a/packages/client/ui/AlertDialog/AlertDialogTrigger.tsx b/packages/client/ui/AlertDialog/AlertDialogTrigger.tsx index 0a20feddfc1..6b2d6149df4 100644 --- a/packages/client/ui/AlertDialog/AlertDialogTrigger.tsx +++ b/packages/client/ui/AlertDialog/AlertDialogTrigger.tsx @@ -1,3 +1,3 @@ import * as AlertDialogPrimitive from '@radix-ui/react-alert-dialog' -const AlertDialogTrigger = AlertDialogPrimitive.Trigger +export const AlertDialogTrigger = AlertDialogPrimitive.Trigger diff --git a/packages/client/utils/__tests__/parseEmailAddressList.test.ts b/packages/client/utils/__tests__/parseEmailAddressList.test.ts index a9778e7bebe..87c83ee894e 100644 --- a/packages/client/utils/__tests__/parseEmailAddressList.test.ts +++ b/packages/client/utils/__tests__/parseEmailAddressList.test.ts @@ -1,7 +1,12 @@ /* eslint-env jest */ import parseEmailAddressList from '../parseEmailAddressList' -const getAddressStr = (res) => res && res.parsedInvitees.map((val) => val.address).join(', ') +type Res = { + parsedInvitees: { + [other: string]: any + }[] +} +const getAddressStr = (res: Res) => res && res.parsedInvitees.map((val) => val.address).join(', ') describe('parseEmailAddressList', () => { it('validates a simple single email', () => { diff --git a/packages/client/utils/getBezierTimePercentGivenDistancePercent.ts b/packages/client/utils/getBezierTimePercentGivenDistancePercent.ts index af2b6ab346b..3b988aec450 100644 --- a/packages/client/utils/getBezierTimePercentGivenDistancePercent.ts +++ b/packages/client/utils/getBezierTimePercentGivenDistancePercent.ts @@ -10,7 +10,12 @@ const bezierLookup = [] as {x: number; y: number}[] const getBezierTimePercentGivenDistancePercent = (threshold: number, bezierCurve: string) => { if (bezierLookup.length === 0) { const re = /\(([^)]+)\)/ - const [x1, y1, x2, y2] = re.exec(bezierCurve)![1].split(',').map(Number) + const [x1, y1, x2, y2] = re.exec(bezierCurve)![1]!.split(',').map(Number) as [ + number, + number, + number, + number + ] // css bezier-curves imply a start of 0,0 and end of 1,1 const x0 = 0 const y0 = 0 @@ -32,9 +37,9 @@ const getBezierTimePercentGivenDistancePercent = (threshold: number, bezierCurve } let x for (let i = 1; i < bezierLookup.length; i++) { - const point = bezierLookup[i] + const point = bezierLookup[i]! if (point.y > threshold) { - x = bezierLookup[i - 1].x + x = bezierLookup[i - 1]!.x break } } diff --git a/packages/client/utils/screenBugs/Bug.ts b/packages/client/utils/screenBugs/Bug.ts index 536d0142887..0ab7411d40d 100644 --- a/packages/client/utils/screenBugs/Bug.ts +++ b/packages/client/utils/screenBugs/Bug.ts @@ -183,7 +183,7 @@ export default class Bug { this.bug.classList.remove('bug-dead') } - animate = (t) => { + animate = (t: any) => { if (!this.animating || !this.alive || !this.active) return this.going = requestAnimationFrame((t) => { this.animate(t) @@ -217,9 +217,9 @@ export default class Bug { this.angle_deg %= 360 if (this.angle_deg < 0) this.angle_deg += 360 - if (Math.abs(this.directions[this.near_edge] - this.angle_deg) > 15) { - const angle1 = this.directions[this.near_edge] - this.angle_deg - const angle2 = 360 - this.angle_deg + this.directions[this.near_edge] + if (Math.abs(this.directions[this.near_edge]! - this.angle_deg) > 15) { + const angle1 = this.directions[this.near_edge]! - this.angle_deg + const angle2 = 360 - this.angle_deg + this.directions[this.near_edge]! this.large_turn_angle_deg = Math.abs(angle1) < Math.abs(angle2) ? angle1 : angle2 this.edge_test_counter = 10 @@ -399,7 +399,7 @@ export default class Bug { let side = Math.round(Math.random() * 4 - 0.5) const d = document const e = d.documentElement - const g = d.getElementsByTagName('body')[0] + const g = d.getElementsByTagName('body')[0]! const windowX = window.innerWidth || e.clientWidth || g.clientWidth const windowY = window.innerHeight || e.clientHeight || g.clientHeight if (side > 3) side = 3 @@ -455,7 +455,7 @@ export default class Bug { let side = Math.round(Math.random() * 4 - 0.5) const d = document const e = d.documentElement - const g = d.getElementsByTagName('body')[0] + const g = d.getElementsByTagName('body')[0]! const windowX = window.innerWidth || e.clientWidth || g.clientWidth const windowY = window.innerHeight || e.clientHeight || g.clientHeight if (side > 3) side = 3 @@ -496,7 +496,7 @@ export default class Bug { const style = {} as Pos const d = document const e = d.documentElement - const g = d.getElementsByTagName('body')[0] + const g = d.getElementsByTagName('body')[0]! const windowX = window.innerWidth || e.clientWidth || g.clientWidth const windowY = window.innerHeight || e.clientHeight || g.clientHeight @@ -529,11 +529,11 @@ export default class Bug { this.drop(deathType) } - drop = (deathType) => { + drop = (deathType: any) => { const startPos = this.bug.top const d = document const e = d.documentElement - const g = d.getElementsByTagName('body')[0] + const g = d.getElementsByTagName('body')[0]! const pos = window.innerHeight || e.clientHeight || g.clientHeight const finalPos = pos - this.options.bugHeight const rotationRate = this.random(0, 20, true) @@ -545,7 +545,7 @@ export default class Bug { }) } - dropping = (t, startPos, finalPos, rotationRate, deathType) => { + dropping = (t: number, startPos: any, finalPos: any, rotationRate: any, deathType: any) => { const elapsedTime = t - this._lastTimestamp! const deltaPos = 0.002 * (elapsedTime * elapsedTime) let newPos = startPos + deltaPos diff --git a/packages/client/utils/screenBugs/BugController.ts b/packages/client/utils/screenBugs/BugController.ts index 132c1796103..330f729ad61 100644 --- a/packages/client/utils/screenBugs/BugController.ts +++ b/packages/client/utils/screenBugs/BugController.ts @@ -131,7 +131,7 @@ class BugDispatch { this.spawnDelay = [] for (let i = 0; i < numBugs; i++) { const delay = this.random(this.options.minDelay, this.options.maxDelay, true) - const thebug = this.bugs[i] + const thebug = this.bugs[i]! // fly the bug onto the page: this.spawnDelay[i] = window.setTimeout(() => { this.options.canFly ? thebug.flyIn() : thebug.walkIn() @@ -152,30 +152,30 @@ class BugDispatch { stop = () => { for (let i = 0; i < this.bugs.length; i++) { if (this.spawnDelay[i]) clearTimeout(this.spawnDelay[i]) - this.bugs[i].stop() + this.bugs[i]!.stop() } } end = () => { for (let i = 0; i < this.bugs.length; i++) { if (this.spawnDelay[i]) clearTimeout(this.spawnDelay[i]) - this.bugs[i].stop() - this.bugs[i].remove() + this.bugs[i]!.stop() + this.bugs[i]!.remove() } } reset = () => { this.stop() for (let i = 0; i < this.bugs.length; i++) { - this.bugs[i].reset() - this.bugs[i].walkIn() + this.bugs[i]!.reset() + this.bugs[i]!.walkIn() } } killAll = () => { for (let i = 0; i < this.bugs.length; i++) { if (this.spawnDelay[i]) clearTimeout(this.spawnDelay[i]) - this.bugs[i].die() + this.bugs[i]!.die() } } @@ -209,13 +209,13 @@ class BugDispatch { } const numBugs = this.bugs.length for (let i = 0; i < numBugs; i++) { - const pos = this.bugs[i].getPos() + const pos = this.bugs[i]!.getPos() if (pos) { if ( Math.abs(pos.top - posy) + Math.abs(pos.left - posx) < this.options.eventDistanceToBug && - !this.bugs[i].flyperiodical + !this.bugs[i]!.flyperiodical ) { - this.near_bug(this.bugs[i]) + this.near_bug(this.bugs[i]!) } } } @@ -233,7 +233,7 @@ class BugDispatch { let mode = this.options.mouseOver if (mode === 'random') { - mode = this.modes[this.random(0, this.modes.length - 1, true)] + mode = this.modes[this.random(0, this.modes.length - 1, true)]! } if (mode === 'fly') { diff --git a/packages/embedder/EMBEDDER_JOB_PRIORITY.ts b/packages/embedder/EMBEDDER_JOB_PRIORITY.ts new file mode 100644 index 00000000000..a54e4b5c67d --- /dev/null +++ b/packages/embedder/EMBEDDER_JOB_PRIORITY.ts @@ -0,0 +1,6 @@ +export const EMBEDDER_JOB_PRIORITY = { + MEETING: 40, + DEFAULT: 50, + TOPIC_HISTORY: 80, + NEW_MODEL: 90 +} as const diff --git a/packages/embedder/EmbeddingsJobQueueStream.ts b/packages/embedder/EmbeddingsJobQueueStream.ts new file mode 100644 index 00000000000..64f7de936dd --- /dev/null +++ b/packages/embedder/EmbeddingsJobQueueStream.ts @@ -0,0 +1,73 @@ +import {Selectable, sql} from 'kysely' +import ms from 'ms' +import sleep from 'parabol-client/utils/sleep' +import 'parabol-server/initSentry' +import getKysely from 'parabol-server/postgres/getKysely' +import {DB} from 'parabol-server/postgres/pg' +import RootDataLoader from '../server/dataloader/RootDataLoader' +import {processJob} from './processJob' +import {Logger} from '../server/utils/Logger' +import {EmbeddingsTableName} from './ai_models/AbstractEmbeddingsModel' + +export type DBJob = Selectable +export type EmbedJob = DBJob & { + jobType: 'embed' + jobData: { + embeddingsMetadataId: number + model: EmbeddingsTableName + } +} +export type RerankJob = DBJob & {jobType: 'rerank'; jobData: {discussionIds: string[]}} +export type Job = EmbedJob | RerankJob + +export class EmbeddingsJobQueueStream implements AsyncIterableIterator { + [Symbol.asyncIterator]() { + return this + } + dataLoader = new RootDataLoader({maxBatchSize: 1000}) + async next(): Promise> { + const pg = getKysely() + const getJob = (isFailed: boolean) => { + return pg + .with( + (cte) => cte('ids').materialized(), + (db) => + db + .selectFrom('EmbeddingsJobQueue') + .select('id') + .orderBy(['priority']) + .$if(!isFailed, (db) => db.where('state', '=', 'queued')) + .$if(isFailed, (db) => + db.where('state', '=', 'failed').where('retryAfter', '<', new Date()) + ) + .limit(1) + .forUpdate() + .skipLocked() + ) + .updateTable('EmbeddingsJobQueue') + .set({state: 'running', startAt: new Date()}) + .where('id', '=', sql`ANY(SELECT id FROM ids)`) + .returningAll() + .executeTakeFirst() + } + const job = (await getJob(false)) || (await getJob(true)) + if (!job) { + Logger.log('JobQueueStream: no jobs found') + // queue is empty, so sleep for a while + await sleep(ms('1m')) + return this.next() + } + + const isSuccessful = await processJob(job as Job, this.dataLoader) + if (isSuccessful) { + await pg.deleteFrom('EmbeddingsJobQueue').where('id', '=', job.id).executeTakeFirstOrThrow() + } + return {done: false, value: job as Job} + } + return() { + return Promise.resolve({done: true as const, value: undefined}) + } + throw(error: any) { + return Promise.resolve({done: true, value: error}) + } +} diff --git a/packages/embedder/README.md b/packages/embedder/README.md index fc3fc68f335..36bb8e2ea50 100644 --- a/packages/embedder/README.md +++ b/packages/embedder/README.md @@ -3,27 +3,14 @@ This service builds embedding vectors for semantic search and for other AI/ML use cases. It does so by: -1. Updating a list of all possible items to create embedding vectors for and - storing that list in the `EmbeddingsMetadata` table -2. Adding these items in batches to the `EmbeddingsJobQueue` table and a redis - priority queue called `embedder:queue` -3. Allowing one or more parallel embedding services to calculate embedding - vectors (EmbeddingJobQueue states transistion from `queued` -> `embedding`, - then `embedding` -> [deleting the `EmbeddingJobQueue` row] - - In addition to deleteing the `EmbeddingJobQueue` row, when a job completes - successfully: - - - A row is added to the model table with the embedding vector; the - `EmbeddingMetadataId` field on this row points the appropriate - metadata row on `EmbeddingsMetadata` - - The `EmbeddingsMetadata.models` array is updated with the name of the - table that the embedding has been generated for - -4. This process repeats forever using a silly polling loop - -In the future, it would be wonderful to enhance this service such that it were -event driven. +1. Homogenizes different types of data into a single `EmbeddingsMetadata` table +2. Each new row in `EmbeddingsMetadata` creates a new row in `EmbeddingsJobQueue` for each model +3. Uses PG to pick a job from the queue and sets the job from `queued` -> `embedding`, + then `embedding` -> [deleting the `EmbeddingJobQueue` row] +4. Embedding involves creating a `fullText` from the work item and then a vector from that `fullText` +5. New jobs to add metadata are sent via redis streams from the GQL Executor +6. If embedding fails, the application increments the `retryCount` and increases the `retryAfter` if a retry is desired +7. If a job gets stalled, a process that runs every 5 minutes will look for jobs older than 5 minutes and reset them to `queued` ## Prerequisites @@ -37,10 +24,9 @@ The predeploy script checks for an environment variable The Embedder service takes no arguments and is controlled by the following environment variables, here given with example configuration: -- `AI_EMBEDDER_ENABLE`: enable/disable the embedder service from - performing work, or sleeping indefinitely +- `AI_EMBEDDER_WORKERS`: How many workers should simultaneously pick jobs from the queue. If less than 1, disabled. -`AI_EMBEDDER_ENABLED='true'` +`AI_EMBEDDER_WORKERS='1'` - `AI_EMBEDDING_MODELS`: JSON configuration for which embedding models are enabled. Each model in the array will be instantiated by @@ -69,3 +55,10 @@ environment variables, here given with example configuration: The Embedder service is stateless and takes no arguments. Multiple instances of the service may be started in order to match embedding load, or to catch up on history more quickly. + +## Resources + +### PG as a Job Queue + +- https://leontrolski.github.io/postgres-as-queue.html +- https://www.2ndquadrant.com/en/blog/what-is-select-skip-locked-for-in-postgresql-9-5/ diff --git a/packages/embedder/addEmbeddingsMetadata.ts b/packages/embedder/addEmbeddingsMetadata.ts new file mode 100644 index 00000000000..214fecc0409 --- /dev/null +++ b/packages/embedder/addEmbeddingsMetadata.ts @@ -0,0 +1,15 @@ +import {addEmbeddingsMetadataForRetrospectiveDiscussionTopic} from './addEmbeddingsMetadataForRetrospectiveDiscussionTopic' +import {MessageToEmbedder} from './custom' + +export const addEmbeddingsMetadata = async ({objectTypes, ...options}: MessageToEmbedder) => { + return Promise.all( + objectTypes.map((type) => { + switch (type) { + case 'retrospectiveDiscussionTopic': + return addEmbeddingsMetadataForRetrospectiveDiscussionTopic(options) + default: + throw new Error(`Invalid object type: ${type}`) + } + }) + ) +} diff --git a/packages/embedder/addEmbeddingsMetadataForRetrospectiveDiscussionTopic.ts b/packages/embedder/addEmbeddingsMetadataForRetrospectiveDiscussionTopic.ts new file mode 100644 index 00000000000..cd4489e3942 --- /dev/null +++ b/packages/embedder/addEmbeddingsMetadataForRetrospectiveDiscussionTopic.ts @@ -0,0 +1,144 @@ +import {ExpressionOrFactory, SqlBool, sql} from 'kysely' +import getRethink from 'parabol-server/database/rethinkDriver' +import {RDatum} from 'parabol-server/database/stricterR' +import getKysely from 'parabol-server/postgres/getKysely' +import {DB} from 'parabol-server/postgres/pg' +import {Logger} from 'parabol-server/utils/Logger' +import {EMBEDDER_JOB_PRIORITY} from './EMBEDDER_JOB_PRIORITY' +import getModelManager from './ai_models/ModelManager' +import {EmbedderOptions} from './custom' + +interface DiscussionMeta { + id: string + teamId: string + createdAt: Date +} + +const validateDiscussions = async (discussions: (DiscussionMeta & {meetingId: string})[]) => { + const r = await getRethink() + if (discussions.length === 0) return discussions + // Exclude discussions that belong to an unfinished meeting + const meetingIds = [...new Set(discussions.map(({meetingId}) => meetingId))] + const endedMeetingIds = await r + .table('NewMeeting') + .getAll(r.args(meetingIds), {index: 'id'}) + .filter((row: RDatum) => row('endedAt').default(null).ne(null))('id') + .distinct() + .run() + const endedMeetingIdsSet = new Set(endedMeetingIds) + return discussions.filter(({meetingId}) => endedMeetingIdsSet.has(meetingId)) +} + +const insertDiscussionsIntoMetadata = async (discussions: DiscussionMeta[], priority: number) => { + const pg = getKysely() + const metadataRows = discussions.map(({id, teamId, createdAt}) => ({ + refId: id, + objectType: 'retrospectiveDiscussionTopic' as const, + teamId, + // Not techincally updatedAt since discussions are be updated after they get created + refUpdatedAt: createdAt + })) + if (!metadataRows[0]) return + + const modelManager = getModelManager() + const tableNames = [...modelManager.embeddingModels.keys()] + return ( + pg + .with('Insert', (qc) => + qc + .insertInto('EmbeddingsMetadata') + .values(metadataRows) + .onConflict((oc) => oc.doNothing()) + .returning('id') + ) + // create n*m rows for n models & m discussions + .with('Metadata', (qc) => + qc + .selectFrom('Insert') + .fullJoin( + sql<{model: string}>`UNNEST(ARRAY[${sql.join(tableNames)}])`.as('model'), + (join) => join.onTrue() + ) + .select(['id', 'model']) + ) + .insertInto('EmbeddingsJobQueue') + .columns(['jobType', 'priority', 'jobData']) + .expression(({selectFrom}) => + selectFrom('Metadata').select(({lit, fn, ref}) => [ + sql.lit('embed').as('jobType'), + lit(priority).as('priority'), + fn('json_build_object', [ + sql.lit('embeddingsMetadataId'), + ref('Metadata.id'), + sql.lit('model'), + ref('Metadata.model') + ]).as('jobData') + ]) + ) + .execute() + ) +} + +export const addEmbeddingsMetadataForRetrospectiveDiscussionTopic = async ({ + startAt, + endAt, + meetingId +}: EmbedderOptions) => { + // load up the metadata table will all discussion topics that are a part of meetings ended within the given date range + const pg = getKysely() + if (meetingId) { + const discussions = await pg + .selectFrom('Discussion') + .select(['id', 'teamId', 'createdAt']) + .where('meetingId', '=', meetingId) + .execute() + await insertDiscussionsIntoMetadata(discussions, EMBEDDER_JOB_PRIORITY.MEETING) + return + } + // PG only accepts 65K parameters (inserted columns * number of rows + query params). Make the batches as big as possible + const PG_MAX_PARAMS = 65535 + const QUERY_PARAMS = 10 + const METADATA_COLS_PER_ROW = 4 + const BATCH_SIZE = Math.floor((PG_MAX_PARAMS - QUERY_PARAMS) / METADATA_COLS_PER_ROW) + const pgStartAt = startAt || new Date(0) + const pgEndAt = (endAt || new Date('4000-01-01')).getTime() / 1000 + + let curEndAt = pgEndAt + let curEndId = '' + for (let i = 0; i < 1e6; i++) { + // preserve microsecond resolution to keep timestamps equal + // so we can use the ID as a tiebreaker when count(createdAt) > BATCH_SIZE + const pgTime = sql`to_timestamp(${curEndAt})` + const lessThanTimeOrId: ExpressionOrFactory = curEndId + ? ({eb}) => + eb('createdAt', '<', pgTime).or(eb('createdAt', '=', pgTime).and('id', '>', curEndId)) + : ({eb}) => eb('createdAt', '<=', pgTime) + const discussions = await pg + .selectFrom('Discussion') + .select([ + 'id', + 'teamId', + 'createdAt', + 'meetingId', + sql`extract(epoch from "createdAt")`.as('createdAtEpoch') + ]) + .where('createdAt', '>', pgStartAt) + .where(lessThanTimeOrId) + .where('discussionTopicType', '=', 'reflectionGroup') + .orderBy('createdAt', 'desc') + .orderBy('id') + .limit(BATCH_SIZE) + .execute() + const earliestDiscussionInBatch = discussions.at(-1) + if (!earliestDiscussionInBatch) break + const {createdAtEpoch, id} = earliestDiscussionInBatch + curEndId = curEndAt === createdAtEpoch ? id : '' + curEndAt = createdAtEpoch + const validDiscussions = await validateDiscussions(discussions) + await insertDiscussionsIntoMetadata(validDiscussions, EMBEDDER_JOB_PRIORITY.TOPIC_HISTORY) + const jsTime = new Date(createdAtEpoch * 1000) + Logger.log( + `Inserted ${validDiscussions.length}/${discussions.length} discussions in metadata ending at ${jsTime}` + ) + } +} diff --git a/packages/embedder/ai_models/AbstractEmbeddingsModel.ts b/packages/embedder/ai_models/AbstractEmbeddingsModel.ts new file mode 100644 index 00000000000..f9fb669737d --- /dev/null +++ b/packages/embedder/ai_models/AbstractEmbeddingsModel.ts @@ -0,0 +1,154 @@ +import {sql} from 'kysely' +import getKysely from 'parabol-server/postgres/getKysely' +import {DB} from 'parabol-server/postgres/pg' +import isValid from '../../server/graphql/isValid' +import {Logger} from '../../server/utils/Logger' +import {EMBEDDER_JOB_PRIORITY} from '../EMBEDDER_JOB_PRIORITY' +import {ISO6391} from '../iso6393To1' +import {AbstractModel} from './AbstractModel' + +export interface EmbeddingModelParams { + embeddingDimensions: number + maxInputTokens: number + tableSuffix: string + languages: ISO6391[] +} +export type EmbeddingsTableName = `Embeddings_${string}` +export type EmbeddingsTable = Extract + +export abstract class AbstractEmbeddingsModel extends AbstractModel { + readonly embeddingDimensions: number + readonly maxInputTokens: number + readonly tableName: EmbeddingsTableName + readonly languages: ISO6391[] + constructor(modelId: string, url: string) { + super(url) + const modelParams = this.constructModelParams(modelId) + this.embeddingDimensions = modelParams.embeddingDimensions + this.languages = modelParams.languages + this.maxInputTokens = modelParams.maxInputTokens + this.tableName = `Embeddings_${modelParams.tableSuffix}` + } + protected abstract constructModelParams(modelId: string): EmbeddingModelParams + abstract getEmbedding(content: string, retries?: number): Promise + + abstract getTokens(content: string): Promise + + async chunkText(content: string) { + const tokens = await this.getTokens(content) + if (tokens instanceof Error) return tokens + const isFullTextTooBig = tokens.length > this.maxInputTokens + if (!isFullTextTooBig) return [content] + + for (let i = 0; i < 3; i++) { + const tokensPerWord = (4 + i) / 3 + const chunks = this.splitText(content, tokensPerWord) + const chunkLengths = await Promise.all( + chunks.map(async (chunk) => { + const chunkTokens = await this.getTokens(chunk) + if (chunkTokens instanceof Error) return chunkTokens + return chunkTokens.length + }) + ) + const firstError = chunkLengths.find( + (chunkLength): chunkLength is Error => chunkLength instanceof Error + ) + if (firstError) return firstError + + const validChunks = chunkLengths.filter(isValid) + if (validChunks.every((chunkLength) => chunkLength <= this.maxInputTokens)) { + return chunks + } + } + return new Error(`Text is too long and could not be split into chunks. Is it english?`) + } + // private because result must still be too long to go into model. Must verify with getTokens + private splitText(content: string, tokensPerWord = 4 / 3) { + // it's actually 4 / 3, but don't want to chance a failed split + const WORD_LIMIT = Math.floor(this.maxInputTokens / tokensPerWord) + const chunks: string[] = [] + const delimiters = ['\n\n', '\n', '.', ' '] + const countWords = (text: string) => text.trim().split(/\s+/).length + const splitOnDelimiter = (text: string, delimiter: string) => { + const sections = text.split(delimiter) + for (let i = 0; i < sections.length; i++) { + const section = sections[i]! + const sectionWordCount = countWords(section) + if (sectionWordCount < WORD_LIMIT) { + // try to merge this section with the last one + const previousSection = chunks.at(-1) + if (previousSection) { + const combinedChunks = `${previousSection}${delimiter}${section}` + const mergedWordCount = countWords(combinedChunks) + if (mergedWordCount < WORD_LIMIT) { + chunks[chunks.length - 1] = combinedChunks + continue + } + } + chunks.push(section) + } else { + const nextDelimiter = delimiters[delimiters.indexOf(delimiter) + 1]! + splitOnDelimiter(section, nextDelimiter) + } + } + } + splitOnDelimiter(content.trim(), delimiters[0]!) + return chunks + } + + async createEmbeddingsForModel() { + Logger.log(`Queueing EmbeddingsMetadata into EmbeddingsJobQueue for ${this.tableName}`) + const pg = getKysely() + await pg + .insertInto('EmbeddingsJobQueue') + .columns(['jobData', 'priority']) + .expression(({selectFrom}) => + selectFrom('EmbeddingsMetadata') + .select(({fn, lit}) => [ + fn('json_build_object', [ + sql.lit('model'), + sql.lit(this.tableName), + sql.lit('embeddingsMetadataId'), + 'id' + ]).as('jobData'), + lit(EMBEDDER_JOB_PRIORITY.NEW_MODEL).as('priority') + ]) + .where('language', 'in', this.languages) + ) + .onConflict((oc) => oc.doNothing()) + .execute() + } + async createTable() { + const pg = getKysely() + const hasTable = + ( + await sql`SELECT 1 FROM ${sql.id('pg_catalog', 'pg_tables')} WHERE ${sql.id( + 'tablename' + )} = ${this.tableName}`.execute(pg) + ).rows.length > 0 + if (hasTable) return + const vectorDimensions = this.embeddingDimensions + Logger.log(`ModelManager: creating ${this.tableName} with ${vectorDimensions} dimensions`) + await sql` + DO $$ + BEGIN + CREATE TABLE IF NOT EXISTS ${sql.id(this.tableName)} ( + "id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + "embedText" TEXT, + "embedding" vector(${sql.raw(vectorDimensions.toString())}), + "embeddingsMetadataId" INTEGER UNIQUE NOT NULL, + "chunkNumber" SMALLINT, + UNIQUE("embeddingsMetadataId", "chunkNumber"), + FOREIGN KEY ("embeddingsMetadataId") + REFERENCES "EmbeddingsMetadata"("id") + ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS "idx_${sql.raw(this.tableName)}_embedding_vector_cosign_ops" + ON ${sql.id(this.tableName)} + USING hnsw ("embedding" vector_cosine_ops); + END + $$; + `.execute(pg) + await this.createEmbeddingsForModel() + } +} diff --git a/packages/embedder/ai_models/AbstractGenerationModel.ts b/packages/embedder/ai_models/AbstractGenerationModel.ts new file mode 100644 index 00000000000..ac0223e6eb6 --- /dev/null +++ b/packages/embedder/ai_models/AbstractGenerationModel.ts @@ -0,0 +1,25 @@ +import {AbstractModel} from './AbstractModel' + +export interface GenerationOptions { + maxNewTokens?: number + seed?: number + stop?: string + temperature?: number + topK?: number + topP?: number +} +export interface GenerationModelParams { + maxInputTokens: number +} + +export abstract class AbstractGenerationModel extends AbstractModel { + readonly maxInputTokens: number + constructor(modelId: string, url: string) { + super(url) + const modelParams = this.constructModelParams(modelId) + this.maxInputTokens = modelParams.maxInputTokens + } + + protected abstract constructModelParams(modelId: string): GenerationModelParams + abstract summarize(content: string, options: GenerationOptions): Promise +} diff --git a/packages/embedder/ai_models/AbstractModel.ts b/packages/embedder/ai_models/AbstractModel.ts index b57d220cd35..322476c552f 100644 --- a/packages/embedder/ai_models/AbstractModel.ts +++ b/packages/embedder/ai_models/AbstractModel.ts @@ -1,73 +1,15 @@ -export interface ModelConfig { - model: string - url: string -} - -export interface EmbeddingModelConfig extends ModelConfig { - tableSuffix: string -} - -export interface GenerationModelConfig extends ModelConfig {} - export abstract class AbstractModel { - public readonly url?: string + public readonly url: string - constructor(config: ModelConfig) { - this.url = this.normalizeUrl(config.url) + constructor(url: string) { + this.url = this.normalizeUrl(url) } // removes a trailing slash from the inputUrl - private normalizeUrl(inputUrl: string | undefined) { - if (!inputUrl) return undefined + private normalizeUrl(inputUrl: string) { const regex = /[/]+$/ return inputUrl.replace(regex, '') } } -export interface EmbeddingModelParams { - embeddingDimensions: number - maxInputTokens: number - tableSuffix: string -} - -export abstract class AbstractEmbeddingsModel extends AbstractModel { - readonly embeddingDimensions: number - readonly maxInputTokens: number - readonly tableName: string - constructor(config: EmbeddingModelConfig) { - super(config) - const modelParams = this.constructModelParams(config) - this.embeddingDimensions = modelParams.embeddingDimensions - this.maxInputTokens = modelParams.maxInputTokens - this.tableName = `Embeddings_${modelParams.tableSuffix}` - } - protected abstract constructModelParams(config: EmbeddingModelConfig): EmbeddingModelParams - abstract getEmbedding(content: string): Promise -} - -export interface GenerationModelParams { - maxInputTokens: number -} - -export interface GenerationOptions { - maxNewTokens?: number - seed?: number - stop?: string - temperature?: number - topK?: number - topP?: number -} - -export abstract class AbstractGenerationModel extends AbstractModel { - readonly maxInputTokens: number - constructor(config: GenerationModelConfig) { - super(config) - const modelParams = this.constructModelParams(config) - this.maxInputTokens = modelParams.maxInputTokens - } - - protected abstract constructModelParams(config: GenerationModelConfig): GenerationModelParams - abstract summarize(content: string, options: GenerationOptions): Promise -} - export default AbstractModel diff --git a/packages/embedder/ai_models/ModelManager.ts b/packages/embedder/ai_models/ModelManager.ts index bf6888378c8..bbbfa4f6273 100644 --- a/packages/embedder/ai_models/ModelManager.ts +++ b/packages/embedder/ai_models/ModelManager.ts @@ -1,155 +1,93 @@ -import {Kysely, sql} from 'kysely' - -import { - AbstractEmbeddingsModel, - AbstractGenerationModel, - EmbeddingModelConfig, - GenerationModelConfig, - ModelConfig -} from './AbstractModel' +import {AbstractEmbeddingsModel, EmbeddingsTableName} from './AbstractEmbeddingsModel' +import {AbstractGenerationModel} from './AbstractGenerationModel' import OpenAIGeneration from './OpenAIGeneration' import TextEmbeddingsInference from './TextEmbeddingsInference' import TextGenerationInference from './TextGenerationInference' -interface ModelManagerConfig { - embeddingModels: EmbeddingModelConfig[] - generationModels: GenerationModelConfig[] -} +type EmbeddingsModelType = 'text-embeddings-inference' +type GenerationModelType = 'openai' | 'text-generation-inference' -export type EmbeddingsModelType = 'text-embeddings-inference' -export type GenerationModelType = 'openai' | 'text-generation-inference' +export interface ModelConfig { + model: `${EmbeddingsModelType | GenerationModelType}:${string}` + url: string +} export class ModelManager { - embeddingModels: AbstractEmbeddingsModel[] - embeddingModelsMapByTable: {[key: string]: AbstractEmbeddingsModel} - generationModels: AbstractGenerationModel[] - - private isValidConfig( - maybeConfig: Partial - ): maybeConfig is ModelManagerConfig { - if (!maybeConfig.embeddingModels || !Array.isArray(maybeConfig.embeddingModels)) { - throw new Error('Invalid configuration: embedding_models is missing or not an array') + embeddingModels: Map + generationModels: Map + + private parseModelEnvVars(envVar: 'AI_EMBEDDING_MODELS' | 'AI_GENERATION_MODELS'): ModelConfig[] { + const envValue = process.env[envVar] + if (!envValue) return [] + let models + try { + models = JSON.parse(envValue) + } catch (e) { + throw new Error(`Invalid Env Var: ${envVar}. Must be a valid JSON`) } - if (!maybeConfig.generationModels || !Array.isArray(maybeConfig.generationModels)) { - throw new Error('Invalid configuration: summarization_models is missing or not an array') - } - - maybeConfig.embeddingModels.forEach((model: ModelConfig) => { - this.isValidModelConfig(model) - }) - maybeConfig.generationModels.forEach((model: ModelConfig) => { - this.isValidModelConfig(model) - }) - - return true - } - - private isValidModelConfig(model: ModelConfig): model is ModelConfig { - if (typeof model.model !== 'string') { - throw new Error('Invalid ModelConfig: model field should be a string') + if (!Array.isArray(models)) { + throw new Error(`Invalid Env Var: ${envVar}. Must be an array`) } - if (model.url !== undefined && typeof model.url !== 'string') { - throw new Error('Invalid ModelConfig: url field should be a string') - } - - return true - } - - constructor(config: ModelManagerConfig) { - // Validate configuration - this.isValidConfig(config) - - // Initialize embeddings models - this.embeddingModelsMapByTable = {} - this.embeddingModels = config.embeddingModels.map((modelConfig) => { - const [modelType] = modelConfig.model.split(':') as [EmbeddingsModelType, string] - - switch (modelType) { - case 'text-embeddings-inference': { - const embeddingsModel = new TextEmbeddingsInference(modelConfig) - this.embeddingModelsMapByTable[embeddingsModel.tableName] = embeddingsModel - return embeddingsModel + const properties = ['model', 'url'] + models.forEach((model, idx) => { + properties.forEach((prop) => { + if (typeof model[prop] !== 'string') { + throw new Error(`Invalid Env Var: ${envVar}. Invalid "${prop}" at index ${idx}`) } - default: - throw new Error(`unsupported embeddings model '${modelType}'`) - } + }) }) + return models + } - // Initialize summarization models - this.generationModels = config.generationModels.map((modelConfig) => { - const [modelType, _] = modelConfig.model.split(':') as [GenerationModelType, string] - - switch (modelType) { - case 'openai': { - return new OpenAIGeneration(modelConfig) + constructor() { + // Initialize embeddings models + const embeddingConfig = this.parseModelEnvVars('AI_EMBEDDING_MODELS') + this.embeddingModels = new Map( + embeddingConfig.map((modelConfig) => { + const {model, url} = modelConfig + const [modelType, modelId] = model.split(':') as [EmbeddingsModelType, string] + switch (modelType) { + case 'text-embeddings-inference': { + const embeddingsModel = new TextEmbeddingsInference(modelId, url) + return [embeddingsModel.tableName, embeddingsModel] + } + default: + throw new Error(`unsupported embeddings model '${modelType}'`) } - case 'text-generation-inference': { - return new TextGenerationInference(modelConfig) + }) + ) + + // Initialize generation models + const generationConfig = this.parseModelEnvVars('AI_GENERATION_MODELS') + this.generationModels = new Map( + generationConfig.map((modelConfig) => { + const {model, url} = modelConfig + const [modelType, modelId] = model.split(':') as [GenerationModelType, string] + switch (modelType) { + case 'openai': { + return [modelId, new OpenAIGeneration(modelId, url)] + } + case 'text-generation-inference': { + return [modelId, new TextGenerationInference(modelId, url)] + } + default: + throw new Error(`unsupported generation model '${modelType}'`) } - default: - throw new Error(`unsupported summarization model '${modelType}'`) - } - }) + }) + ) } - async maybeCreateTables(pg: Kysely) { - const maybePromises = this.embeddingModels.map(async (embeddingsModel) => { - const tableName = embeddingsModel.tableName - const hasTable = - ( - await sql`SELECT 1 FROM ${sql.id('pg_catalog', 'pg_tables')} WHERE ${sql.id( - 'tablename' - )} = ${tableName}`.execute(pg) - ).rows.length > 0 - if (hasTable) return undefined - const vectorDimensions = embeddingsModel.embeddingDimensions - console.log(`ModelManager: creating ${tableName} with ${vectorDimensions} dimensions`) - const query = sql` - DO $$ - BEGIN - CREATE TABLE IF NOT EXISTS ${sql.id(tableName)} ( - "id" INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, - "embedText" TEXT, - "embedding" vector(${sql.raw(vectorDimensions.toString())}), - "embeddingsMetadataId" INTEGER NOT NULL, - FOREIGN KEY ("embeddingsMetadataId") - REFERENCES "EmbeddingsMetadata"("id") - ON DELETE CASCADE - ); - CREATE INDEX IF NOT EXISTS "idx_${sql.raw(tableName)}_embedding_vector_cosign_ops" - ON ${sql.id(tableName)} - USING hnsw ("embedding" vector_cosine_ops); - END $$; - - ` - return query.execute(pg) - }) - Promise.all(maybePromises) + async maybeCreateTables() { + return Promise.all([...this.embeddingModels].map(([, model]) => model.createTable())) } } let modelManager: ModelManager | undefined export function getModelManager() { - if (modelManager) return modelManager - const {AI_EMBEDDING_MODELS, AI_GENERATION_MODELS} = process.env - const config: ModelManagerConfig = { - embeddingModels: [], - generationModels: [] - } - try { - config.embeddingModels = AI_EMBEDDING_MODELS && JSON.parse(AI_EMBEDDING_MODELS) - } catch (e) { - throw new Error(`Invalid AI_EMBEDDING_MODELS .env JSON: ${e}`) + if (!modelManager) { + modelManager = new ModelManager() } - try { - config.generationModels = AI_GENERATION_MODELS && JSON.parse(AI_GENERATION_MODELS) - } catch (e) { - throw new Error(`Invalid AI_GENERATION_MODELS .env JSON: ${e}`) - } - - modelManager = new ModelManager(config) - return modelManager } diff --git a/packages/embedder/ai_models/OpenAIGeneration.ts b/packages/embedder/ai_models/OpenAIGeneration.ts index b5614b608c5..2bd97a32822 100644 --- a/packages/embedder/ai_models/OpenAIGeneration.ts +++ b/packages/embedder/ai_models/OpenAIGeneration.ts @@ -1,12 +1,9 @@ import OpenAI from 'openai' import { AbstractGenerationModel, - GenerationModelConfig, GenerationModelParams, GenerationOptions -} from './AbstractModel' - -const MAX_REQUEST_TIME_S = 3 * 60 +} from './AbstractGenerationModel' export type ModelId = 'gpt-3.5-turbo-0125' | 'gpt-4-turbo-preview' @@ -21,16 +18,12 @@ const modelIdDefinitions: Record = { } } -function isValidModelId(object: any): object is ModelId { - return Object.keys(modelIdDefinitions).includes(object) -} - export class OpenAIGeneration extends AbstractGenerationModel { private openAIApi: OpenAI | null - private modelId: ModelId + private modelId!: ModelId - constructor(config: GenerationModelConfig) { - super(config) + constructor(modelId: string, url: string) { + super(modelId, url) if (!process.env.OPEN_AI_API_KEY) { this.openAIApi = null return @@ -75,19 +68,10 @@ export class OpenAIGeneration extends AbstractGenerationModel { throw e } } - protected constructModelParams(config: GenerationModelConfig): GenerationModelParams { - const modelConfigStringSplit = config.model.split(':') - if (modelConfigStringSplit.length != 2) { - throw new Error('OpenAIGeneration model string must be colon-delimited and len 2') - } - - const maybeModelId = modelConfigStringSplit[1] - if (!isValidModelId(maybeModelId)) - throw new Error(`OpenAIGeneration model id unknown: ${maybeModelId}`) - - this.modelId = maybeModelId - - return modelIdDefinitions[maybeModelId] + protected constructModelParams(modelId: string): GenerationModelParams { + const modelParams = modelIdDefinitions[modelId as keyof typeof modelIdDefinitions] + if (!modelParams) throw new Error(`Unknown modelId ${modelId} for OpenAIGeneration`) + return modelParams } } diff --git a/packages/embedder/ai_models/TextEmbeddingsInference.ts b/packages/embedder/ai_models/TextEmbeddingsInference.ts index 549fadcd6fd..1a2a02596b3 100644 --- a/packages/embedder/ai_models/TextEmbeddingsInference.ts +++ b/packages/embedder/ai_models/TextEmbeddingsInference.ts @@ -1,70 +1,86 @@ -import {AbstractEmbeddingsModel, EmbeddingModelConfig, EmbeddingModelParams} from './AbstractModel' -import fetchWithRetry from './helpers/fetchWithRetry' - -const MAX_REQUEST_TIME_S = 3 * 60 - +import createClient from 'openapi-fetch' +import sleep from 'parabol-client/utils/sleep' +import type {paths} from '../textEmbeddingsnterface' +import {AbstractEmbeddingsModel, EmbeddingModelParams} from './AbstractEmbeddingsModel' export type ModelId = 'BAAI/bge-large-en-v1.5' | 'llmrails/ember-v1' const modelIdDefinitions: Record = { 'BAAI/bge-large-en-v1.5': { embeddingDimensions: 1024, maxInputTokens: 512, - tableSuffix: 'bge_l_en_1p5' + tableSuffix: 'bge_l_en_1p5', + languages: ['en'] }, 'llmrails/ember-v1': { embeddingDimensions: 1024, maxInputTokens: 512, - tableSuffix: 'ember_1' + tableSuffix: 'ember_1', + languages: ['en'] } } -function isValidModelId(object: any): object is ModelId { - return Object.keys(modelIdDefinitions).includes(object) -} - export class TextEmbeddingsInference extends AbstractEmbeddingsModel { - constructor(config: EmbeddingModelConfig) { - super(config) + client: ReturnType> + constructor(modelId: string, url: string) { + super(modelId, url) + this.client = createClient({baseUrl: this.url}) } - public async getEmbedding(content: string) { - const fetchOptions = { - body: JSON.stringify({inputs: content}), - deadline: new Date(new Date().getTime() + MAX_REQUEST_TIME_S * 1000), - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json; charset=utf-8' - }, - method: 'POST' + async getTokens(content: string) { + try { + const {data, error} = await this.client.POST('/tokenize', { + body: {inputs: content, add_special_tokens: true}, + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json; charset=utf-8' + } + }) + if (error) return new Error(error.error) + return data[0]!.map(({id}) => id) + } catch (e) { + return e instanceof Error ? e : new Error(e as string) } + } + async decodeTokens(inputIds: number[]) { try { - const res = await fetchWithRetry(`${this.url}/embed`, fetchOptions) - const listOfVectors = (await res.json()) as Array - if (!listOfVectors) - throw new Error('TextEmbeddingsInference.getEmbeddings(): listOfVectors is undefined') - if (listOfVectors.length !== 1 || !listOfVectors[0]) - throw new Error( - `TextEmbeddingsInference.getEmbeddings(): listOfVectors list length !== 1 (length: ${listOfVectors.length})` - ) - return listOfVectors[0] + const {data, error} = await this.client.POST('/decode', { + body: {ids: inputIds}, + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json; charset=utf-8' + } + }) + if (error) return new Error(error.error) + return data } catch (e) { - console.log(`TextEmbeddingsInference.getEmbeddings() timeout: `, e) - throw e + return e instanceof Error ? e : new Error(e as string) } } - - protected constructModelParams(config: EmbeddingModelConfig): EmbeddingModelParams { - const modelConfigStringSplit = config.model.split(':') - if (modelConfigStringSplit.length != 2) { - throw new Error('TextGenerationInference model string must be colon-delimited and len 2') + public async getEmbedding(content: string, retries = 5): Promise { + try { + const {data, error, response} = await this.client.POST('/embed', { + body: {inputs: content}, + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json; charset=utf-8' + } + }) + if (error) { + if (response.status !== 429 || retries < 1) return new Error(error.error) + await sleep(2000) + return this.getEmbedding(content, retries - 1) + } + return data[0]! + } catch (e) { + return e instanceof Error ? e : new Error(e as string) } + } - if (!this.url) throw new Error('TextGenerationInferenceSummarizer model requires url') - const maybeModelId = modelConfigStringSplit[1] - if (!isValidModelId(maybeModelId)) - throw new Error(`TextGenerationInference model id unknown: ${maybeModelId}`) - return modelIdDefinitions[maybeModelId] + protected constructModelParams(modelId: string): EmbeddingModelParams { + const modelParams = modelIdDefinitions[modelId as keyof typeof modelIdDefinitions] + if (!modelParams) throw new Error(`Unknown modelId ${modelId} for TextEmbeddingsInference`) + return modelParams } } diff --git a/packages/embedder/ai_models/TextGenerationInference.ts b/packages/embedder/ai_models/TextGenerationInference.ts index bcf1daa6303..96d7fdde89f 100644 --- a/packages/embedder/ai_models/TextGenerationInference.ts +++ b/packages/embedder/ai_models/TextGenerationInference.ts @@ -1,9 +1,8 @@ import { AbstractGenerationModel, - GenerationModelConfig, GenerationModelParams, GenerationOptions -} from './AbstractModel' +} from './AbstractGenerationModel' import fetchWithRetry from './helpers/fetchWithRetry' const MAX_REQUEST_TIME_S = 3 * 60 @@ -16,13 +15,9 @@ const modelIdDefinitions: Record = { } } -function isValidModelId(object: any): object is ModelId { - return Object.keys(modelIdDefinitions).includes(object) -} - export class TextGenerationInference extends AbstractGenerationModel { - constructor(config: GenerationModelConfig) { - super(config) + constructor(modelId: string, url: string) { + super(modelId, url) } async summarize(content: string, options: GenerationOptions) { @@ -51,7 +46,6 @@ export class TextGenerationInference extends AbstractGenerationModel { } try { - // console.log(`TextGenerationInference.summarize(): summarizing from ${this.url}/generate`) const res = await fetchWithRetry(`${this.url}/generate`, fetchOptions) const json = await res.json() if (!json || !json.generated_text) @@ -62,17 +56,10 @@ export class TextGenerationInference extends AbstractGenerationModel { throw e } } - protected constructModelParams(config: GenerationModelConfig): GenerationModelParams { - const modelConfigStringSplit = config.model.split(':') - if (modelConfigStringSplit.length != 2) { - throw new Error('TextGenerationInference model string must be colon-delimited and len 2') - } - - if (!this.url) throw new Error('TextGenerationInferenceSummarizer model requires url') - const maybeModelId = modelConfigStringSplit[1] - if (!isValidModelId(maybeModelId)) - throw new Error(`TextGenerationInference model id unknown: ${maybeModelId}`) - return modelIdDefinitions[maybeModelId] + protected constructModelParams(modelId: string): GenerationModelParams { + const modelParams = modelIdDefinitions[modelId as keyof typeof modelIdDefinitions] + if (!modelParams) throw new Error(`Unknown modelId ${modelId} for TextGenerationInference`) + return modelParams } } diff --git a/packages/embedder/custom.d.ts b/packages/embedder/custom.d.ts new file mode 100644 index 00000000000..6640974c6a9 --- /dev/null +++ b/packages/embedder/custom.d.ts @@ -0,0 +1,11 @@ +import type {DB} from '../server/postgres/pg' + +export type EmbeddingObjectType = DB['EmbeddingsMetadata']['objectType'] + +export interface MessageToEmbedder { + objectTypes: EmbeddingObjectType[] + startAt?: Date + endAt?: Date + meetingId?: string +} +export type EmbedderOptions = Omit diff --git a/packages/embedder/embedder.ts b/packages/embedder/embedder.ts index f6762013d08..7971ba4f717 100644 --- a/packages/embedder/embedder.ts +++ b/packages/embedder/embedder.ts @@ -1,42 +1,18 @@ -import {Insertable} from 'kysely' import tracer from 'dd-trace' -import Redlock, {RedlockAbortSignal} from 'redlock' - +import EmbedderChannelId from 'parabol-client/shared/gqlIds/EmbedderChannelId' import 'parabol-server/initSentry' -import getKysely from 'parabol-server/postgres/getKysely' -import {DB} from 'parabol-server/postgres/pg' -import {refreshRetroDiscussionTopicsMeta as refreshRetroDiscussionTopicsMeta} from './indexing/retrospectiveDiscussionTopic' -import {orgIdsWithFeatureFlag} from './indexing/orgIdsWithFeatureFlag' -import getModelManager, {ModelManager} from './ai_models/ModelManager' -import {countWords} from './indexing/countWords' -import {createEmbeddingTextFrom} from './indexing/createEmbeddingTextFrom' -import { - selectJobQueueItemById, - selectMetadataByJobQueueId, - updateJobState -} from './indexing/embeddingsTablesOps' -import {selectMetaToQueue} from './indexing/embeddingsTablesOps' -import {insertNewJobs} from './indexing/embeddingsTablesOps' -import {completeJobTxn} from './indexing/embeddingsTablesOps' -import {getRootDataLoader} from './indexing/getRootDataLoader' -import {getRedisClient} from './indexing/getRedisClient' - -/* - * TODO List - * - [ ] implement a clean-up function that re-queues items that haven't transitioned - * to a completed state, or that failed - */ - -export type DBInsert = { - [K in keyof DB]: Insertable -} - -const POLLING_PERIOD_SEC = 60 // How often do we try to grab the lock and re-index? -const Q_MAX_LENGTH = 100 // How many EmbeddingIndex items do we batch in redis? -const WORD_COUNT_TO_TOKEN_RATIO = 3.0 / 2 // We multiple the word count by this to estimate token count - -const {AI_EMBEDDER_ENABLED} = process.env -const {SERVER_ID} = process.env +import {Logger} from 'parabol-server/utils/Logger' +import RedisInstance from 'parabol-server/utils/RedisInstance' +import {Tuple} from '../client/types/generics' +import RedisStream from '../gql-executor/RedisStream' +import {EmbeddingsJobQueueStream} from './EmbeddingsJobQueueStream' +import {addEmbeddingsMetadata} from './addEmbeddingsMetadata' +import getModelManager from './ai_models/ModelManager' +import {MessageToEmbedder} from './custom' +import {establishPrimaryEmbedder} from './establishPrimaryEmbedder' +import {importHistoricalMetadata} from './importHistoricalMetadata' +import {mergeAsyncIterators} from './mergeAsyncIterators' +import {resetStalledJobs} from './resetStalledJobs' tracer.init({ service: `embedder`, @@ -46,207 +22,89 @@ tracer.init({ }) tracer.use('pg') -const refreshMetadata = async () => { - const dataLoader = getRootDataLoader() - await refreshRetroDiscussionTopicsMeta(dataLoader) - // In the future, other sorts of objects to index could be added here... +const parseEmbedderMessage = (message: string): MessageToEmbedder => { + const {startAt, endAt, ...input} = JSON.parse(message) + return { + ...input, + startAt: startAt ? new Date(startAt) : undefined, + endAt: endAt ? new Date(endAt) : undefined + } } -const maybeQueueMetadataItems = async (modelManager: ModelManager) => { - const redisClient = getRedisClient() - const queueLength = await redisClient.zcard('embedder:queue') - if (queueLength >= Q_MAX_LENGTH) return - const itemCountToQueue = Q_MAX_LENGTH - queueLength - const modelTables = modelManager.embeddingModels.map((m) => m.tableName) - const orgIds = await orgIdsWithFeatureFlag() - - // For each configured embedding model, select rows from EmbeddingsMetadata - // that haven't been calculated nor exist in the EmbeddingsJobQueue yet - // - // Notes: - // * `em.models @> ARRAY[v.model]` is an indexed query - // * I don't love all overrides, I wish there was a better way - // see: https://github.com/kysely-org/kysely/issues/872 - - const batchToQueue = await selectMetaToQueue(modelTables, orgIds, itemCountToQueue) - if (!batchToQueue.length) { - console.log(`embedder: no new items to queue`) +const run = async () => { + const SERVER_ID = process.env.SERVER_ID + if (!SERVER_ID) throw new Error('env.SERVER_ID is required') + const embedderChannel = EmbedderChannelId.join(SERVER_ID) + const NUM_WORKERS = parseInt(process.env.AI_EMBEDDER_WORKERS!) + if (!(NUM_WORKERS > 0)) { + Logger.log('env.AI_EMBEDDER_WORKERS is < 0. Embedder will not run.') return } - const ejqHash: { - [key: string]: { - refUpdatedAt: Date - } - } = {} - const makeKey = (item: {objectType: string; refId: string}) => `${item.objectType}:${item.refId}` - - const ejqValues = batchToQueue.map((item) => { - ejqHash[makeKey(item)] = { - refUpdatedAt: item.refUpdatedAt - } - return { - objectType: item.objectType, - refId: item.refId as string, - model: item.model, - state: 'queued' as const - } - }) - - const ejqRows = await insertNewJobs(ejqValues) - - ejqRows.forEach((item) => { - const {refUpdatedAt} = ejqHash[makeKey(item)]! - const score = new Date(refUpdatedAt).getTime() - redisClient.zadd('embedder:queue', score, item.id) - }) - - console.log(`embedder: queued ${batchToQueue.length} items`) -} - -const dequeueAndEmbedUntilEmpty = async (modelManager: ModelManager) => { - const dataLoader = getRootDataLoader() - const redisClient = getRedisClient() - while (true) { - const maybeRedisQItem = await redisClient.zpopmax('embedder:queue', 1) - if (maybeRedisQItem.length < 2) return // Q is empty, all done! - - const [id, _] = maybeRedisQItem - if (!id) { - console.log(`embedder: de-queued undefined item from embedder:queue`) - continue - } - const jobQueueId = parseInt(id, 10) - const jobQueueItem = await selectJobQueueItemById(jobQueueId) - if (!jobQueueItem) { - console.log(`embedder: unable to fetch EmbeddingsJobQueue.id = ${id}`) - continue - } - - const metadata = await selectMetadataByJobQueueId(jobQueueId) - if (!metadata) { - await updateJobState(jobQueueId, 'failed', { - stateMessage: `unable to fetch metadata by EmbeddingsJobQueue.id = ${id}` - }) - continue - } - - let fullText = metadata?.fullText - try { - if (!fullText) { - fullText = await createEmbeddingTextFrom(jobQueueItem, dataLoader) - } - } catch (e) { - await updateJobState(jobQueueId, 'failed', { - stateMessage: `unable to create embedding text: ${e}` - }) - continue - } + const redis = new RedisInstance(`embedder_${SERVER_ID}`) + const primaryLock = await establishPrimaryEmbedder(redis) + const modelManager = getModelManager() + let streams: AsyncIterableIterator | undefined = undefined + const kill = () => { + primaryLock?.release() + streams?.return?.() + process.exit() + } + process.on('SIGTERM', kill) + process.on('SIGINT', kill) + if (primaryLock) { + // only 1 worker needs to perform these on startup + await modelManager.maybeCreateTables() + await importHistoricalMetadata() + resetStalledJobs() + } - const wordCount = countWords(fullText) + const onMessage = async (_channel: string, message: string) => { + const parsedMessage = parseEmbedderMessage(message) + await addEmbeddingsMetadata(parsedMessage) + } - const embeddingModel = modelManager.embeddingModelsMapByTable[jobQueueItem.model] - if (!embeddingModel) { - await updateJobState(jobQueueId, 'failed', { - stateMessage: `embedding model ${jobQueueItem.model} not available` - }) - continue - } - const itemKey = `${jobQueueItem.objectType}:${jobQueueItem.refId}` - const modelTable = embeddingModel.tableName + // subscribe to consumer group + try { + await redis.xgroup( + 'CREATE', + 'embedMetadataStream', + 'embedMetadataConsumerGroup', + '$', + 'MKSTREAM' + ) + } catch (e) { + // stream already exists + } - let embedText = fullText - const maxInputTokens = embeddingModel.maxInputTokens - // we're using word count as an appoximation of tokens - if (wordCount * WORD_COUNT_TO_TOKEN_RATIO > maxInputTokens) { - try { - const generator = modelManager.generationModels[0] // use 1st generator - if (!generator) throw new Error(`Generator unavailable`) - console.log(`embedder: ...summarizing ${itemKey} for ${modelTable}`) - embedText = await generator.summarize(fullText, {maxNewTokens: maxInputTokens}) - } catch (e) { - await updateJobState(jobQueueId, 'failed', { - stateMessage: `unable to summarize long embed text: ${e}` - }) + const messageStream = new RedisStream( + 'embedMetadataStream', + 'embedMetadataConsumerGroup', + embedderChannel + ) + + // Assume 3 workers for type safety, but it doesn't really matter at runtime + const jobQueueStreams = Array.from( + {length: NUM_WORKERS}, + () => new EmbeddingsJobQueueStream() + ) as Tuple + + Logger.log(`\n⚡⚡⚡️️ Server ID: ${SERVER_ID}. Embedder is ready ⚡⚡⚡️️️`) + + streams = mergeAsyncIterators([messageStream, ...jobQueueStreams]) + for await (const [idx, message] of streams) { + switch (idx) { + case 0: + onMessage('', message) + continue + default: + Logger.log(`Worker ${idx} finished job ${message.id}`) continue - } - } - // console.log(`embedText: ${embedText}`) - - let embeddingVector: number[] - try { - embeddingVector = await embeddingModel.getEmbedding(embedText) - } catch (e) { - await updateJobState(jobQueueId, 'failed', { - stateMessage: `unable to get embeddings: ${e}` - }) - continue } - - // complete job, do the following atomically - // (1) update EmbeddingsMetadata to reflect model completion - // (2) upsert model table row with embedding - // (3) delete EmbeddingsJobQueue row - await completeJobTxn(modelTable, jobQueueId, metadata, fullText, embedText, embeddingVector) - console.log(`embedder: completed ${itemKey} -> ${modelTable}`) } -} - -const tick = async (modelManager: ModelManager) => { - console.log(`embedder: tick`) - const redisClient = getRedisClient() - const redlock = new Redlock([redisClient], { - driftFactor: 0.01, - retryCount: 10, - retryDelay: 250, - retryJitter: 50, - automaticExtensionThreshold: 500 - }) - - await redlock - .using(['embedder:lock'], 10000, async (signal: RedlockAbortSignal) => { - console.log(`embedder: acquired index queue lock`) - // N.B. one of the many benefits of using redlock is the using() interface - // will automatically extend the lock if these operations exceed the - // original redis timeout time - await refreshMetadata() - await maybeQueueMetadataItems(modelManager) - - if (signal.aborted) { - // Not certain which conditions this would happen, it would - // happen after operations took place, so nothing much to do here. - console.log('embedder: lock was lost!') - } - }) - .catch((err: string) => { - // Handle errors (including lock acquisition errors) - console.error('embedder: an error occurred ', err) - }) - console.log('embedder: index queue lock released') - - // get the highest priority item and embed it - await dequeueAndEmbedUntilEmpty(modelManager) - - setTimeout(() => tick(modelManager), POLLING_PERIOD_SEC * 1000) -} - -function parseEnvBoolean(envVarValue: string | undefined): boolean { - return envVarValue === 'true' -} -const run = async () => { - console.log(`embedder: run()`) - const embedderEnabled = parseEnvBoolean(AI_EMBEDDER_ENABLED) - const modelManager = getModelManager() - if (embedderEnabled && modelManager) { - const pg = getKysely() - await modelManager.maybeCreateTables(pg) - console.log(`\n⚡⚡⚡️️ Server ID: ${SERVER_ID}. Embedder is ready ⚡⚡⚡️️️`) - tick(modelManager) - } else { - console.log(`embedder: no valid configuration (check AI_EMBEDDER_ENABLED in .env)`) - // exit - } + // On graceful shutdown + Logger.log('Streaming Complete. Goodbye!') } run() diff --git a/packages/embedder/establishPrimaryEmbedder.ts b/packages/embedder/establishPrimaryEmbedder.ts new file mode 100644 index 00000000000..72f349d655f --- /dev/null +++ b/packages/embedder/establishPrimaryEmbedder.ts @@ -0,0 +1,17 @@ +import ms from 'ms' +import RedisInstance from 'parabol-server/utils/RedisInstance' +import Redlock from 'redlock' + +export const establishPrimaryEmbedder = async (redis: RedisInstance) => { + const redlock = new Redlock([redis], {retryCount: 0}) + const MAX_TIME_BETWEEN_WORKER_STARTUPS = ms('5s') + try { + const primaryWorkerLock = await redlock.acquire( + [`embedder_isPrimary_${process.env.npm_package_version}`], + MAX_TIME_BETWEEN_WORKER_STARTUPS + ) + return primaryWorkerLock + } catch { + return undefined + } +} diff --git a/packages/embedder/importHistoricalMetadata.ts b/packages/embedder/importHistoricalMetadata.ts new file mode 100644 index 00000000000..0b805f18888 --- /dev/null +++ b/packages/embedder/importHistoricalMetadata.ts @@ -0,0 +1,16 @@ +import {EmbeddingObjectType} from './custom' +import {importHistoricalRetrospectiveDiscussionTopic} from './importHistoricalRetrospectiveDiscussionTopic' + +export const importHistoricalMetadata = async () => { + const OBJECT_TYPES: EmbeddingObjectType[] = ['retrospectiveDiscussionTopic'] + return Promise.all( + OBJECT_TYPES.map(async (objectType) => { + switch (objectType) { + case 'retrospectiveDiscussionTopic': + return importHistoricalRetrospectiveDiscussionTopic() + default: + throw new Error(`Invalid object type: ${objectType}`) + } + }) + ) +} diff --git a/packages/embedder/importHistoricalRetrospectiveDiscussionTopic.ts b/packages/embedder/importHistoricalRetrospectiveDiscussionTopic.ts new file mode 100644 index 00000000000..2469327a18a --- /dev/null +++ b/packages/embedder/importHistoricalRetrospectiveDiscussionTopic.ts @@ -0,0 +1,37 @@ +import getKysely from 'parabol-server/postgres/getKysely' +import {Logger} from 'parabol-server/utils/Logger' +import {addEmbeddingsMetadataForRetrospectiveDiscussionTopic} from './addEmbeddingsMetadataForRetrospectiveDiscussionTopic' + +// Check to see if the oldest discussion topic exists in the metadata table +// If not, get the date of the oldest discussion topic in the metadata table and import all items before that date +export const importHistoricalRetrospectiveDiscussionTopic = async () => { + const pg = getKysely() + const isEarliestMetadataImported = await pg + .selectFrom('EmbeddingsMetadata') + .select('id') + .where(({eb, selectFrom}) => + eb( + 'EmbeddingsMetadata.refId', + '=', + selectFrom('Discussion') + .select('Discussion.id') + .where('discussionTopicType', '=', 'reflectionGroup') + .orderBy(['createdAt', 'id']) + .limit(1) + ) + ) + .limit(1) + .executeTakeFirst() + + if (isEarliestMetadataImported) return + const earliestImportedDiscussion = await pg + .selectFrom('EmbeddingsMetadata') + .select(['id', 'refUpdatedAt', 'refId']) + .where('objectType', '=', 'retrospectiveDiscussionTopic') + .orderBy('refUpdatedAt') + .limit(1) + .executeTakeFirst() + const endAt = earliestImportedDiscussion?.refUpdatedAt ?? undefined + Logger.log(`Importing discussion history up to ${endAt || 'now'}`) + return addEmbeddingsMetadataForRetrospectiveDiscussionTopic({endAt}) +} diff --git a/packages/embedder/indexing/countWords.ts b/packages/embedder/indexing/countWords.ts deleted file mode 100644 index 75dae3effa2..00000000000 --- a/packages/embedder/indexing/countWords.ts +++ /dev/null @@ -1,17 +0,0 @@ -export function countWords(text: string) { - let count = 0 - let inWord = false - - for (const char of text) { - if (/\w/.test(char)) { - if (!inWord) { - count++ - inWord = true - } - } else { - inWord = false - } - } - - return count -} diff --git a/packages/embedder/indexing/createEmbeddingTextFrom.ts b/packages/embedder/indexing/createEmbeddingTextFrom.ts index 9d6e66b60e7..9fb3cceda80 100644 --- a/packages/embedder/indexing/createEmbeddingTextFrom.ts +++ b/packages/embedder/indexing/createEmbeddingTextFrom.ts @@ -1,15 +1,17 @@ import {Selectable} from 'kysely' import {DB} from 'parabol-server/postgres/pg' -import {DataLoaderWorker} from 'parabol-server/graphql/graphql' -import {createText as createTextFromRetrospectiveDiscussionTopic} from './retrospectiveDiscussionTopic' +import RootDataLoader from 'parabol-server/dataloader/RootDataLoader' +import {createTextFromRetrospectiveDiscussionTopic} from './retrospectiveDiscussionTopic' export const createEmbeddingTextFrom = async ( - item: Selectable, - dataLoader: DataLoaderWorker -): Promise => { - switch (item.objectType) { + embeddingsMetadata: Selectable, + dataLoader: RootDataLoader +) => { + switch (embeddingsMetadata.objectType) { case 'retrospectiveDiscussionTopic': - return createTextFromRetrospectiveDiscussionTopic(item, dataLoader) + return createTextFromRetrospectiveDiscussionTopic(embeddingsMetadata.refId, dataLoader) + default: + throw new Error(`Unexcepted objectType: ${embeddingsMetadata.objectType}`) } } diff --git a/packages/embedder/indexing/embeddingsTablesOps.ts b/packages/embedder/indexing/embeddingsTablesOps.ts deleted file mode 100644 index c74eb709708..00000000000 --- a/packages/embedder/indexing/embeddingsTablesOps.ts +++ /dev/null @@ -1,198 +0,0 @@ -import {Insertable, Selectable, Updateable, sql} from 'kysely' -import getKysely from 'parabol-server/postgres/getKysely' -import {DB} from 'parabol-server/postgres/pg' -import {DBInsert} from '../embedder' -import {RawBuilder} from 'kysely' -import numberVectorToString from './numberVectorToString' - -function unnestedArray(maybeArray: T[] | T): RawBuilder { - let a: T[] = Array.isArray(maybeArray) ? maybeArray : [maybeArray] - return sql`unnest(ARRAY[${sql.join(a)}]::varchar[])` -} - -export const selectJobQueueItemById = async ( - id: number -): Promise | undefined> => { - const pg = getKysely() - return pg.selectFrom('EmbeddingsJobQueue').selectAll().where('id', '=', id).executeTakeFirst() -} -export const selectMetadataByJobQueueId = async ( - id: number -): Promise | undefined> => { - const pg = getKysely() - return pg - .selectFrom('EmbeddingsMetadata as em') - .selectAll() - .leftJoin('EmbeddingsJobQueue as ejq', (join) => - join.onRef('em.objectType', '=', 'ejq.objectType').onRef('em.refId', '=', 'ejq.refId') - ) - .where('ejq.id', '=', id) - .executeTakeFirstOrThrow() -} - -// For each configured embedding model, select rows from EmbeddingsMetadata -// that haven't been calculated nor exist in the EmbeddingsJobQueue yet -// -// Notes: -// * `em.models @> ARRAY[v.model]` is an indexed query -// * I don't love all overrides, I wish there was a better way -// see: https://github.com/kysely-org/kysely/issues/872 -export async function selectMetaToQueue( - configuredModels: string[], - orgIds: any[], - itemCountToQueue: number -) { - const pg = getKysely() - const maybeMetaToQueue = (await pg - .selectFrom('EmbeddingsMetadata as em') - .selectAll('em') - .leftJoinLateral(unnestedArray(configuredModels).as('model'), (join) => join.onTrue()) - .leftJoin('Team as t', 'em.teamId', 't.id') - .select('model' as any) - .where(({eb, not, or, and, exists, selectFrom}) => - and([ - or([ - not(eb('em.models', '@>', sql`ARRAY[${sql.ref('model')}]::varchar[]` as any) as any), - eb('em.models' as any, 'is', null) - ]), - not( - exists( - selectFrom('EmbeddingsJobQueue as ejq') - .select('ejq.id') - .whereRef('em.objectType', '=', 'ejq.objectType') - .whereRef('em.refId', '=', 'ejq.refId') - .whereRef('ejq.model', '=', 'model' as any) - ) - ), - eb('t.orgId', 'in', orgIds) - ]) - ) - .limit(itemCountToQueue) - .execute()) as unknown as Selectable[] - - type MetadataToQueue = Selectable< - Omit & { - refId: NonNullable - } & {model: string} - > - - return maybeMetaToQueue.filter( - (item) => item.refId !== null && item.refId !== undefined - ) as MetadataToQueue[] -} - -export const updateJobState = async ( - id: number, - state: Updateable['state'], - jobQueueFields: Updateable = {} -) => { - const pg = getKysely() - const jobQueueColumns: Updateable = { - ...jobQueueFields, - state - } - if (state === 'failed') console.log(`embedder: failed job ${id}, ${jobQueueFields.stateMessage}`) - return pg - .updateTable('EmbeddingsJobQueue') - .set(jobQueueColumns) - .where('id', '=', id) - .executeTakeFirstOrThrow() -} - -export function insertNewJobs(ejqValues: Insertable[]) { - const pg = getKysely() - return pg - .insertInto('EmbeddingsJobQueue') - .values(ejqValues) - .returning(['id', 'objectType', 'refId']) - .execute() -} - -// complete job, do the following atomically -// (1) update EmbeddingsMetadata to reflect model completion -// (2) upsert model table row with embedding -// (3) delete EmbeddingsJobQueue row -export function completeJobTxn( - modelTable: string, - jobQueueId: number, - metadata: Updateable, - fullText: string, - embedText: string, - embeddingVector: number[] -) { - const pg = getKysely() - return pg.transaction().execute(async (trx) => { - // get fields to update correct metadata row - const jobQueueItem = await trx - .selectFrom('EmbeddingsJobQueue') - .select(['objectType', 'refId', 'model']) - .where('id', '=', jobQueueId) - .executeTakeFirstOrThrow() - - // (1) update metadata row - const metadataColumnsToUpdate: { - models: RawBuilder - fullText?: string | null | undefined - } = { - // update models as a set - models: sql`( -SELECT array_agg(DISTINCT value) -FROM ( - SELECT unnest(COALESCE("models", '{}')) AS value - UNION - SELECT unnest(ARRAY[${modelTable}]::VARCHAR[]) AS value -) AS combined_values -)` - } - - if (metadata?.fullText !== fullText) { - metadataColumnsToUpdate.fullText = fullText - } - - const updatedMetadata = await trx - .updateTable('EmbeddingsMetadata') - .set(metadataColumnsToUpdate) - .where('objectType', '=', jobQueueItem.objectType) - .where('refId', '=', jobQueueItem.refId) - .returning(['id']) - .executeTakeFirstOrThrow() - - // (2) upsert into model table - await trx - .insertInto(modelTable as any) - .values({ - embedText: fullText !== embedText ? embedText : null, - embedding: numberVectorToString(embeddingVector), - embeddingsMetadataId: updatedMetadata.id - }) - .onConflict((oc) => - oc.column('id').doUpdateSet((eb) => ({ - embedText: eb.ref('excluded.embedText'), - embeddingsMetadataId: eb.ref('excluded.embeddingsMetadataId') - })) - ) - .executeTakeFirstOrThrow() - - // (3) delete completed job queue item - return await trx - .deleteFrom('EmbeddingsJobQueue') - .where('id', '=', jobQueueId) - .executeTakeFirstOrThrow() - }) -} -export async function upsertEmbeddingsMetaRows( - embeddingsMetaRows: DBInsert['EmbeddingsMetadata'][] -) { - const pg = getKysely() - return pg - .insertInto('EmbeddingsMetadata') - .values(embeddingsMetaRows) - .onConflict((oc) => - oc.columns(['objectType', 'refId']).doUpdateSet((eb) => ({ - objectType: eb.ref('excluded.objectType'), - refId: eb.ref('excluded.refId'), - refUpdatedAt: eb.ref('excluded.refUpdatedAt') - })) - ) - .execute() -} diff --git a/packages/embedder/indexing/failJob.ts b/packages/embedder/indexing/failJob.ts new file mode 100644 index 00000000000..17d293b49a9 --- /dev/null +++ b/packages/embedder/indexing/failJob.ts @@ -0,0 +1,17 @@ +import getKysely from 'parabol-server/postgres/getKysely' +import {Logger} from 'parabol-server/utils/Logger' + +export const failJob = async (jobId: number, stateMessage: string, retryAfter?: Date | null) => { + const pg = getKysely() + Logger.log(`embedder: failed job ${jobId}, ${stateMessage}`) + await pg + .updateTable('EmbeddingsJobQueue') + .set((eb) => ({ + state: 'failed', + stateMessage, + retryCount: eb('retryCount', '+', 1), + retryAfter: retryAfter || null + })) + .where('id', '=', jobId) + .executeTakeFirstOrThrow() +} diff --git a/packages/embedder/indexing/getRedisClient.ts b/packages/embedder/indexing/getRedisClient.ts deleted file mode 100644 index 7aaf65be33c..00000000000 --- a/packages/embedder/indexing/getRedisClient.ts +++ /dev/null @@ -1,11 +0,0 @@ -import RedisInstance from 'parabol-server/utils/RedisInstance' - -const {SERVER_ID} = process.env - -let redisClient: RedisInstance -export const getRedisClient = () => { - if (!redisClient) { - redisClient = new RedisInstance(`embedder-${SERVER_ID}`) - } - return redisClient -} diff --git a/packages/embedder/indexing/getRootDataLoader.ts b/packages/embedder/indexing/getRootDataLoader.ts deleted file mode 100644 index 304c0c01058..00000000000 --- a/packages/embedder/indexing/getRootDataLoader.ts +++ /dev/null @@ -1,10 +0,0 @@ -import getDataLoader from 'parabol-server/graphql/getDataLoader' -import {DataLoaderWorker} from 'parabol-server/graphql/graphql' - -let rootDataLoader: DataLoaderWorker -export const getRootDataLoader = () => { - if (!rootDataLoader) { - rootDataLoader = getDataLoader() as DataLoaderWorker - } - return rootDataLoader -} diff --git a/packages/embedder/indexing/retrospectiveDiscussionTopic.ts b/packages/embedder/indexing/retrospectiveDiscussionTopic.ts index f31aab74cc2..03af3b510ba 100644 --- a/packages/embedder/indexing/retrospectiveDiscussionTopic.ts +++ b/packages/embedder/indexing/retrospectiveDiscussionTopic.ts @@ -1,26 +1,8 @@ -import {Selectable} from 'kysely' -import prettier from 'prettier' - -import getRethink, {RethinkSchema} from 'parabol-server/database/rethinkDriver' -import {DataLoaderWorker} from 'parabol-server/graphql/graphql' -import getKysely from 'parabol-server/postgres/getKysely' -import {DB} from 'parabol-server/postgres/pg' - +import {RethinkSchema} from 'parabol-server/database/rethinkDriver' import Comment from 'parabol-server/database/types/Comment' -import DiscussStage from 'parabol-server/database/types/DiscussStage' -import MeetingRetrospective, { - isMeetingRetrospective -} from 'parabol-server/database/types/MeetingRetrospective' - -import {upsertEmbeddingsMetaRows} from './embeddingsTablesOps' -import {AnyMeeting} from 'parabol-server/postgres/types/Meeting' - -const BATCH_SIZE = 1000 - -export interface EmbeddingsJobQueueRetrospectiveDiscussionTopic - extends Omit { - objectType: 'retrospectiveDiscussionTopic' -} +import {isMeetingRetrospective} from 'parabol-server/database/types/MeetingRetrospective' +import RootDataLoader from 'parabol-server/dataloader/RootDataLoader' +import prettier from 'prettier' // Here's a generic reprentation of the text generated here: @@ -39,109 +21,14 @@ export interface EmbeddingsJobQueueRetrospectiveDiscussionTopic const IGNORE_COMMENT_USER_IDS = ['parabolAIUser'] -const pg = getKysely() - -export async function refreshRetroDiscussionTopicsMeta(dataLoader: DataLoaderWorker) { - const r = await getRethink() - const {createdAt: newestMeetingDate} = (await r - .table('NewMeeting') - .max({index: 'createdAt'}) - .run()) as unknown as RethinkSchema['NewMeeting']['type'] - const {createdAt: oldestMeetingDate} = (await r - .table('NewMeeting') - .min({index: 'createdAt'}) - .run()) as unknown as RethinkSchema['NewMeeting']['type'] - - const {newestMetaDate} = (await pg - .selectFrom('EmbeddingsMetadata') - .select(pg.fn.max('refUpdatedAt').as('newestMetaDate')) - .where('objectType', '=', 'retrospectiveDiscussionTopic') - .executeTakeFirst()) ?? {newestMetaDate: null} - let startDateTime = newestMetaDate || oldestMeetingDate - - if (startDateTime.getTime() === newestMeetingDate.getTime()) return - - console.log( - `refreshRetroDiscussionTopicsMeta(): ` + - `will consider adding items from ${startDateTime.toISOString()} to ` + - `${newestMeetingDate.toISOString()}` - ) - - let totalAdded = 0 - do { - // Process history in batches. - // - // N.B. We add historical meetings to the EmbeddingsMetadata table here. - // This query will intentionally miss meetings that haven't been completed - // (`summarySentAt` is null). These meetings will need to be added to the - // EmbeddingsMetadata table by a hook that runs when the meetings complete. - const {maxCreatedAt, completedNewMeetings} = await r - .table('NewMeeting') - .between(startDateTime, newestMeetingDate, {rightBound: 'closed', index: 'createdAt'}) - .orderBy({index: 'createdAt'}) - .limit(BATCH_SIZE) - .coerceTo('array') - .do((rows: any) => ({ - maxCreatedAt: r.expr(rows).max('createdAt')('createdAt'), // Then find the max createdAt value - completedNewMeetings: r.expr(rows).filter((r: any) => - r('meetingType') - .eq('retrospective') - .and( - r('endedAt').gt(0), - r - .hasFields('phases') - .and(r('phases').count().gt(0)) - .and( - r('phases') - .filter((phase: any) => phase('phaseType').eq('discuss')) - .filter((phase: any) => - phase.hasFields('stages').and(phase('stages').count().gt(0)) - ) - .count() - .gt(0) - ) - ) - ) - })) - .run() - const embeddingsMetaRows = ( - await Promise.all( - completedNewMeetings.map((m: AnyMeeting) => - newRetroDiscussionTopicsFromNewMeeting(m, dataLoader) - ) - ) - ).flat() - if (embeddingsMetaRows.length > 0) { - await upsertEmbeddingsMetaRows(embeddingsMetaRows) - totalAdded += embeddingsMetaRows.length - console.log( - `refreshRetroDiscussionTopicsMeta(): synced to ${maxCreatedAt.toISOString()}, added` + - ` ${embeddingsMetaRows.length} retrospectiveDiscussionTopics` - ) - } - - // N.B. In the unlikely event that we have >=BATCH_SIZE meetings that end at _exactly_ - // the same timetsamp, this will loop forever. - if ( - startDateTime.getTime() === newestMeetingDate.getTime() && - completedNewMeetings.length < BATCH_SIZE - ) - break - startDateTime = maxCreatedAt - } while (true) - - console.log( - `refreshRetroDiscussionTopicsMeta(): added ${totalAdded} total retrospectiveDiscussionTopics` - ) -} - -async function getPreferredNameByUserId(userId: string, dataLoader: DataLoaderWorker) { +async function getPreferredNameByUserId(userId: string, dataLoader: RootDataLoader) { + if (!userId) return 'Unknown' const user = await dataLoader.get('users').load(userId) return !user ? 'Unknown' : user.preferredName } async function formatThread( - dataLoader: DataLoaderWorker, + dataLoader: RootDataLoader, comments: Comment[], parentId: string | null = null, depth = 0 @@ -156,9 +43,7 @@ async function formatThread( const indent = ' '.repeat(depth + 1) const author = comment.isAnonymous ? 'Anonymous' - : comment.createdBy - ? await getPreferredNameByUserId(comment.createdBy, dataLoader) - : 'Unknown' + : await getPreferredNameByUserId(comment.createdBy, dataLoader) const how = depth === 0 ? 'wrote' : 'replied' const content = comment.plaintextContent const formattedPost = `${indent}- ${author} ${how}, "${content}"\n` @@ -173,60 +58,47 @@ async function formatThread( return formattedComments.join('') } -export const createTextFromNewMeetingDiscussionStage = async ( - newMeeting: MeetingRetrospective, - stageId: string, - dataLoader: DataLoaderWorker, +export const createTextFromRetrospectiveDiscussionTopic = async ( + discussionId: string, + dataLoader: RootDataLoader, textForReranking: boolean = false ) => { - if (!newMeeting) throw 'newMeeting is undefined' - if (!isMeetingRetrospective(newMeeting)) throw 'newMeeting is not retrospective' - if (!newMeeting.templateId) throw 'template is undefined' - const template = await dataLoader.get('meetingTemplates').load(newMeeting.templateId) - if (!template) throw 'template is undefined' - const discussPhase = newMeeting.phases.find((phase) => phase.phaseType === 'discuss') - if (!discussPhase) throw 'newMeeting discuss phase is undefined' - if (!discussPhase.stages) throw 'newMeeting discuss phase has no stages' - const discussStage = discussPhase.stages.find((stage) => stage.id === stageId) as DiscussStage - if (!discussStage) throw 'newMeeting discuss stage not found' - const {summary: discussionSummary} = discussStage.discussionId - ? (await dataLoader.get('discussions').load(discussStage.discussionId)) ?? {summary: null} - : {summary: null} - const r = await getRethink() - if (!discussStage.reflectionGroupId) throw 'newMeeting discuss stage has no reflectionGroupId' - const reflectionGroup = await r - .table('RetroReflectionGroup') - .get(discussStage.reflectionGroupId) - .run() - if (!reflectionGroup.id) throw 'newMeeting reflectionGroup has no id' - const reflections = await r - .table('RetroReflection') - .getAll(reflectionGroup.id, {index: 'reflectionGroupId'}) - .run() + const discussion = await dataLoader.get('discussions').load(discussionId) + if (!discussion) throw new Error(`Discussion not found: ${discussionId}`) + const {discussionTopicId: reflectionGroupId, meetingId, summary: discussionSummary} = discussion + const [newMeeting, reflectionGroup, reflections] = await Promise.all([ + dataLoader.get('newMeetings').load(meetingId), + dataLoader.get('retroReflectionGroups').load(reflectionGroupId), + dataLoader.get('retroReflectionsByGroupId').load(reflectionGroupId) + ]) + if (!isMeetingRetrospective(newMeeting)) throw new Error('Meeting is not a retro') + const {templateId} = newMeeting + const promptIds = [...new Set(reflections.map((r) => r.promptId))] + const [template, ...prompts] = await Promise.all([ + dataLoader.get('meetingTemplates').loadNonNull(templateId), + ...promptIds.map((promptId) => dataLoader.get('reflectPrompts').load(promptId)) + ]) + let markdown = '' - if (!textForReranking) + if (!textForReranking) { markdown = - `A topic "${reflectionGroup.title}" was discussed during ` + + `A topic "${reflectionGroup?.title ?? ''}" was discussed during ` + `the meeting "${newMeeting.name}" that followed the "${template.name}" template.\n` + `\n` - const prompts = await dataLoader.get('reflectPrompts').loadMany(promptIds) + } + for (const prompt of prompts) { - if (!prompt || prompt instanceof Error) continue if (!textForReranking) { markdown += `Participants were prompted with, "${prompt.question}` if (prompt.description) markdown += `: ${prompt.description}` markdown += `".\n` } - if (newMeeting.disableAnonymity) { - for (const reflection of reflections.filter((r) => r.promptId === prompt.id)) { - const author = await getPreferredNameByUserId(reflection.creatorId, dataLoader) - markdown += ` - ${author} wrote, "${reflection.plaintextContent}"\n` - } - } else { - for (const reflection of reflections.filter((r) => r.promptId === prompt.id)) { - markdown += ` - Anonymous wrote, "${reflection.plaintextContent}"\n` - } + for (const reflection of reflections.filter((r) => r.promptId === prompt.id)) { + const author = newMeeting.disableAnonymity + ? await getPreferredNameByUserId(reflection.creatorId, dataLoader) + : 'Anonymous' + markdown += ` - ${author} wrote, "${reflection.plaintextContent}"\n` } markdown += `\n` } @@ -251,7 +123,7 @@ export const createTextFromNewMeetingDiscussionStage = async ( if (discussionSummary) { markdown += `Further discussion was made. ` + ` ${discussionSummary}` } else { - const comments = await dataLoader.get('commentsByDiscussionId').load(stageId) + const comments = await dataLoader.get('commentsByDiscussionId').load(discussionId) const sortedComments = comments .map((comment) => { @@ -267,8 +139,8 @@ export const createTextFromNewMeetingDiscussionStage = async ( if (a.threadParentId === b.threadParentId) { return a.threadSortOrder - b.threadSortOrder } - if (a.threadParentId == null) return 1 - if (b.threadParentId == null) return -1 + if (!a.threadParentId) return 1 + if (!b.threadParentId) return -1 return a.threadParentId > b.threadParentId ? 1 : -1 }) as Comment[] @@ -291,25 +163,9 @@ export const createTextFromNewMeetingDiscussionStage = async ( return markdown } -export const createText = async ( - item: Selectable, - dataLoader: DataLoaderWorker -): Promise => { - if (!item.refId) throw 'refId is undefined' - const [newMeetingId, discussionId] = item.refId.split(':') - if (!newMeetingId) throw new Error('newMeetingId cannot be undefined') - if (!discussionId) throw new Error('discussionId cannot be undefined') - const newMeeting = await dataLoader.get('newMeetings').load(newMeetingId) - return createTextFromNewMeetingDiscussionStage( - newMeeting as MeetingRetrospective, - discussionId, - dataLoader - ) -} - export const newRetroDiscussionTopicsFromNewMeeting = async ( newMeeting: RethinkSchema['NewMeeting']['type'], - dataLoader: DataLoaderWorker + dataLoader: RootDataLoader ) => { const discussPhase = newMeeting.phases.find((phase) => phase.phaseType === 'discuss') const orgId = (await dataLoader.get('teams').load(newMeeting.teamId))?.orgId diff --git a/packages/embedder/iso6393To1.ts b/packages/embedder/iso6393To1.ts new file mode 100644 index 00000000000..f1e470e1232 --- /dev/null +++ b/packages/embedder/iso6393To1.ts @@ -0,0 +1,195 @@ +import {ValueOf} from '../client/types/generics' + +/** + * Map of ISO 639-3 codes to ISO 639-1 codes. + * + * @type {Record} + */ +export const iso6393To1 = { + aar: 'aa', + abk: 'ab', + afr: 'af', + aka: 'ak', + amh: 'am', + ara: 'ar', + arg: 'an', + asm: 'as', + ava: 'av', + ave: 'ae', + aym: 'ay', + aze: 'az', + bak: 'ba', + bam: 'bm', + bel: 'be', + ben: 'bn', + bis: 'bi', + bod: 'bo', + bos: 'bs', + bre: 'br', + bul: 'bg', + cat: 'ca', + ces: 'cs', + cha: 'ch', + che: 'ce', + chu: 'cu', + chv: 'cv', + cor: 'kw', + cos: 'co', + cre: 'cr', + cym: 'cy', + dan: 'da', + deu: 'de', + div: 'dv', + dzo: 'dz', + ell: 'el', + eng: 'en', + epo: 'eo', + est: 'et', + eus: 'eu', + ewe: 'ee', + fao: 'fo', + fas: 'fa', + fij: 'fj', + fin: 'fi', + fra: 'fr', + fry: 'fy', + ful: 'ff', + gla: 'gd', + gle: 'ga', + glg: 'gl', + glv: 'gv', + grn: 'gn', + guj: 'gu', + hat: 'ht', + hau: 'ha', + hbs: 'sh', + heb: 'he', + her: 'hz', + hin: 'hi', + hmo: 'ho', + hrv: 'hr', + hun: 'hu', + hye: 'hy', + ibo: 'ig', + ido: 'io', + iii: 'ii', + iku: 'iu', + ile: 'ie', + ina: 'ia', + ind: 'id', + ipk: 'ik', + isl: 'is', + ita: 'it', + jav: 'jv', + jpn: 'ja', + kal: 'kl', + kan: 'kn', + kas: 'ks', + kat: 'ka', + kau: 'kr', + kaz: 'kk', + khm: 'km', + kik: 'ki', + kin: 'rw', + kir: 'ky', + kom: 'kv', + kon: 'kg', + kor: 'ko', + kua: 'kj', + kur: 'ku', + lao: 'lo', + lat: 'la', + lav: 'lv', + lim: 'li', + lin: 'ln', + lit: 'lt', + ltz: 'lb', + lub: 'lu', + lug: 'lg', + mah: 'mh', + mal: 'ml', + mar: 'mr', + mkd: 'mk', + mlg: 'mg', + mlt: 'mt', + mon: 'mn', + mri: 'mi', + msa: 'ms', + mya: 'my', + nau: 'na', + nav: 'nv', + nbl: 'nr', + nde: 'nd', + ndo: 'ng', + nep: 'ne', + nld: 'nl', + nno: 'nn', + nob: 'nb', + nor: 'no', + nya: 'ny', + oci: 'oc', + oji: 'oj', + ori: 'or', + orm: 'om', + oss: 'os', + pan: 'pa', + pli: 'pi', + pol: 'pl', + por: 'pt', + pus: 'ps', + que: 'qu', + roh: 'rm', + ron: 'ro', + run: 'rn', + rus: 'ru', + sag: 'sg', + san: 'sa', + sin: 'si', + slk: 'sk', + slv: 'sl', + sme: 'se', + smo: 'sm', + sna: 'sn', + snd: 'sd', + som: 'so', + sot: 'st', + spa: 'es', + sqi: 'sq', + srd: 'sc', + srp: 'sr', + ssw: 'ss', + sun: 'su', + swa: 'sw', + swe: 'sv', + tah: 'ty', + tam: 'ta', + tat: 'tt', + tel: 'te', + tgk: 'tg', + tgl: 'tl', + tha: 'th', + tir: 'ti', + ton: 'to', + tsn: 'tn', + tso: 'ts', + tuk: 'tk', + tur: 'tr', + twi: 'tw', + uig: 'ug', + ukr: 'uk', + urd: 'ur', + uzb: 'uz', + ven: 've', + vie: 'vi', + vol: 'vo', + wln: 'wa', + wol: 'wo', + xho: 'xh', + yid: 'yi', + yor: 'yo', + zha: 'za', + zho: 'zh', + zul: 'zu' +} as const + +export type ISO6391 = ValueOf diff --git a/packages/embedder/logMemoryUse.ts b/packages/embedder/logMemoryUse.ts new file mode 100644 index 00000000000..afe3259aee5 --- /dev/null +++ b/packages/embedder/logMemoryUse.ts @@ -0,0 +1,10 @@ +// Not for use in prod, but useful for dev +export const logMemoryUse = () => { + const MB = 2 ** 20 + setInterval(() => { + const memoryUsage = process.memoryUsage() + const {rss} = memoryUsage + const usedMB = Math.floor(rss / MB) + console.log('Memory use:', usedMB, 'MB') + }, 10000) +} diff --git a/packages/embedder/mergeAsyncIterators.ts b/packages/embedder/mergeAsyncIterators.ts new file mode 100644 index 00000000000..e274e0cca6f --- /dev/null +++ b/packages/embedder/mergeAsyncIterators.ts @@ -0,0 +1,96 @@ +import {ParseInt} from '../client/types/generics' + +// can remove PromiseCapability after TS v5.4.2 +type PromiseCapability = { + resolve: (value: T) => void + reject: (reason?: any) => void + promise: Promise +} + +type UnYield = T extends IteratorYieldResult ? U : never +type Result> = UnYield>> + +// Promise.race has a memory leak +// To avoid: https://github.com/tc39/proposal-async-iterator-helpers/issues/15#issuecomment-1937011820 +export function mergeAsyncIterators[] | []>( + iterators: T +): AsyncIterableIterator<{[P in keyof T]: [ParseInt<`${P}`>, Result]}[number]> { + return (async function* () { + type ResultThunk = () => [number, Result] + let count = iterators.length as number + let capability: PromiseCapability | undefined + const queuedResults: ResultThunk[] = [] + const getNext = async (idx: number, iterator: T[number]) => { + try { + const next = await iterator.next() + if (next.done) { + if (--count === 0 && capability !== undefined) { + capability.resolve(null) + } + } else { + resolveResult(() => { + void getNext(idx, iterator) + return [idx, next.value] + }) + } + } catch (error) { + resolveResult(() => { + throw error + }) + } + } + const resolveResult = (resultThunk: ResultThunk) => { + if (capability === undefined) { + queuedResults.push(resultThunk) + } else { + capability.resolve(resultThunk) + } + } + + try { + // Begin all iterators + for (const [idx, iterable] of iterators.entries()) { + void getNext(idx, iterable) + } + + // Delegate to iterables as results complete + while (true) { + while (true) { + const nextQueuedResult = queuedResults.shift() + if (nextQueuedResult === undefined) { + break + } else { + yield nextQueuedResult() + } + } + if (count === 0) { + break + } else { + // Promise.withResolvers() is not yet implemented in node + capability = { + resolve: undefined as any, + reject: undefined as any, + promise: undefined as any + } + capability.promise = new Promise((res, rej) => { + capability!.resolve = res + capability!.reject = rej + }) + const nextResult = await capability.promise + if (nextResult === null) { + break + } else { + capability = undefined + yield nextResult() + } + } + } + } catch (err) { + // Unwind remaining iterators on failure + try { + await Promise.all(iterators.map((iterator) => iterator.return?.())) + } catch {} + throw err + } + })() +} diff --git a/packages/embedder/package.json b/packages/embedder/package.json index 47af8737dac..975309bdb62 100644 --- a/packages/embedder/package.json +++ b/packages/embedder/package.json @@ -1,6 +1,6 @@ { "name": "parabol-embedder", - "version": "7.10.0", + "version": "7.24.1", "description": "A service that computes embedding vectors from Parabol objects", "author": "Jordan Husney ", "homepage": "https://github.com/ParabolInc/parabol/tree/master/packages/embedder#readme", @@ -10,6 +10,10 @@ "url": "git+https://github.com/ParabolInc/parabol.git" }, "scripts": { + "lint": "eslint --fix . --ext .ts,.tsx", + "lint:check": "eslint . --ext .ts,.tsx", + "prettier": "prettier --config ../../.prettierrc --write \"**/*.{ts,tsx}\"", + "prettier:check": "prettier --config ../../.prettierrc --check \"**/*.{ts,tsx}\"", "typecheck": "yarn tsc --noEmit -p tsconfig.json" }, "bugs": { @@ -18,14 +22,17 @@ "devDependencies": { "@babel/cli": "7.18.6", "@babel/core": "7.18.6", + "@types/franc": "^5.0.3", "@types/node": "^16.11.62", "babel-plugin-inline-import": "^3.0.0", + "openapi-fetch": "^0.9.3", "sucrase": "^3.32.0", "ts-node-dev": "^1.0.0-pre.44", - "typescript": "4.9.5" + "typescript": "^5.3.3" }, "dependencies": { "dd-trace": "^4.2.0", + "franc-min": "^5.0.0", "redlock": "^5.0.0-beta.2" } } diff --git a/packages/embedder/processJob.ts b/packages/embedder/processJob.ts new file mode 100644 index 00000000000..8723b75de6a --- /dev/null +++ b/packages/embedder/processJob.ts @@ -0,0 +1,13 @@ +import RootDataLoader from 'parabol-server/dataloader/RootDataLoader' +import {Job} from './EmbeddingsJobQueueStream' +import {processJobEmbed} from './processJobEmbed' + +export const processJob = async (job: Job, dataLoader: RootDataLoader) => { + const {jobType} = job + switch (jobType) { + case 'embed': + return processJobEmbed(job, dataLoader) + default: + throw new Error(`Invalid job type: ${jobType}`) + } +} diff --git a/packages/embedder/processJobEmbed.ts b/packages/embedder/processJobEmbed.ts new file mode 100644 index 00000000000..d7fecf2aab0 --- /dev/null +++ b/packages/embedder/processJobEmbed.ts @@ -0,0 +1,101 @@ +import franc from 'franc-min' +import ms from 'ms' +import RootDataLoader from 'parabol-server/dataloader/RootDataLoader' +import getKysely from 'parabol-server/postgres/getKysely' +import {EmbedJob} from './EmbeddingsJobQueueStream' +import {EmbeddingsTable} from './ai_models/AbstractEmbeddingsModel' +import getModelManager from './ai_models/ModelManager' +import {createEmbeddingTextFrom} from './indexing/createEmbeddingTextFrom' +import {failJob} from './indexing/failJob' +import numberVectorToString from './indexing/numberVectorToString' +import {iso6393To1} from './iso6393To1' + +export const processJobEmbed = async (job: EmbedJob, dataLoader: RootDataLoader) => { + const pg = getKysely() + const {id: jobId, retryCount, jobData} = job + const {embeddingsMetadataId, model} = jobData + const modelManager = getModelManager() + + const metadata = await pg + .selectFrom('EmbeddingsMetadata') + .selectAll() + .where('id', '=', embeddingsMetadataId) + .executeTakeFirst() + + if (!metadata) { + await failJob(jobId, `unable to fetch metadata by EmbeddingsJobQueue.id = ${jobId}`) + return + } + + let {fullText, language} = metadata + try { + if (!fullText) { + fullText = await createEmbeddingTextFrom(metadata, dataLoader) + language = iso6393To1[franc(fullText) as keyof typeof iso6393To1] + await pg + .updateTable('EmbeddingsMetadata') + .set({fullText, language}) + .where('id', '=', embeddingsMetadataId) + .execute() + } + } catch (e) { + // get the trace since the error message may be unobvious + console.trace(e) + await failJob(jobId, `unable to create embedding text: ${e}`) + return + } + + const embeddingModel = modelManager.embeddingModels.get(model) + if (!embeddingModel) { + await failJob(jobId, `embedding model ${model} not available`) + return + } + + // Exit successfully, we don't want to fail the job because the language is not supported + if (!embeddingModel.languages.includes(language!)) return true + + const chunks = await embeddingModel.chunkText(fullText) + if (chunks instanceof Error) { + await failJob( + jobId, + `unable to get tokens: ${chunks.message}`, + retryCount < 10 ? new Date(Date.now() + ms('1m')) : null + ) + return + } + // Cannot use summarization strategy if generation model has same context length as embedding model + // We must split the text & not tokens because BERT tokenizer is not trained for linebreaks e.g. \n\n + const isSuccessful = await Promise.all( + chunks.map(async (chunk, chunkNumber) => { + const embeddingVector = await embeddingModel.getEmbedding(chunk) + if (embeddingVector instanceof Error) { + await failJob( + jobId, + `unable to get embeddings: ${embeddingVector.message}`, + retryCount < 10 ? new Date(Date.now() + ms('1m')) : null + ) + return false + } + await pg + // cast to any because these types won't be available in CI + .insertInto(embeddingModel.tableName as EmbeddingsTable) + .values({ + // TODO is the extra space of a null embedText really worth it?! + embedText: chunks.length > 1 ? chunk : null, + embedding: numberVectorToString(embeddingVector), + embeddingsMetadataId, + chunkNumber: chunks.length > 1 ? chunkNumber : null + }) + .onConflict((oc) => + oc.column('embeddingsMetadataId').doUpdateSet((eb) => ({ + embedText: eb.ref('excluded.embedText'), + embedding: eb.ref('excluded.embedding') + })) + ) + .execute() + return true + }) + ) + // Logger.log(`Embedded ${embeddingsMetadataId} -> ${model}`) + return isSuccessful +} diff --git a/packages/embedder/resetStalledJobs.ts b/packages/embedder/resetStalledJobs.ts new file mode 100644 index 00000000000..592b468faee --- /dev/null +++ b/packages/embedder/resetStalledJobs.ts @@ -0,0 +1,18 @@ +import ms from 'ms' +import getKysely from 'parabol-server/postgres/getKysely' + +export const resetStalledJobs = () => { + setInterval(async () => { + const pg = getKysely() + await pg + .updateTable('EmbeddingsJobQueue') + .set((eb) => ({ + state: 'queued', + startAt: null, + retryCount: eb('retryCount', '+', 1), + stateMessage: 'stalled' + })) + .where('startAt', '<', new Date(Date.now() - ms('5m'))) + .execute() + }, ms('5m')) +} diff --git a/packages/embedder/textEmbeddingsnterface.d.ts b/packages/embedder/textEmbeddingsnterface.d.ts new file mode 100644 index 00000000000..618eea7db91 --- /dev/null +++ b/packages/embedder/textEmbeddingsnterface.d.ts @@ -0,0 +1,877 @@ +/** + * This file was auto-generated by openapi-typescript. + * Do not make direct changes to the file. + */ + +/** OneOf type helpers */ +type Without = {[P in Exclude]?: never} +type XOR = T | U extends object ? (Without & U) | (Without & T) : T | U +type OneOf = T extends [infer Only] + ? Only + : T extends [infer A, infer B, ...infer Rest] + ? OneOf<[XOR, ...Rest]> + : never + +export interface paths { + '/decode': { + /** + * Decode input ids + * @description Decode input ids + */ + post: operations['decode'] + } + '/embed': { + /** + * Get Embeddings. Returns a 424 status code if the model is not an embedding model. + * @description Get Embeddings. Returns a 424 status code if the model is not an embedding model. + */ + post: operations['embed'] + } + '/embed_all': { + /** + * Get all Embeddings without Pooling. + * @description Get all Embeddings without Pooling. + * Returns a 424 status code if the model is not an embedding model. + */ + post: operations['embed_all'] + } + '/embed_sparse': { + /** + * Get Sparse Embeddings. Returns a 424 status code if the model is not an embedding model with SPLADE pooling. + * @description Get Sparse Embeddings. Returns a 424 status code if the model is not an embedding model with SPLADE pooling. + */ + post: operations['embed_sparse'] + } + '/embeddings': { + /** + * OpenAI compatible route. Returns a 424 status code if the model is not an embedding model. + * @description OpenAI compatible route. Returns a 424 status code if the model is not an embedding model. + */ + post: operations['openai_embed'] + } + '/health': { + /** + * Health check method + * @description Health check method + */ + get: operations['health'] + } + '/info': { + /** + * Text Embeddings Inference endpoint info + * @description Text Embeddings Inference endpoint info + */ + get: operations['get_model_info'] + } + '/metrics': { + /** + * Prometheus metrics scrape endpoint + * @description Prometheus metrics scrape endpoint + */ + get: operations['metrics'] + } + '/predict': { + /** + * Get Predictions. Returns a 424 status code if the model is not a Sequence Classification model + * @description Get Predictions. Returns a 424 status code if the model is not a Sequence Classification model + */ + post: operations['predict'] + } + '/rerank': { + /** + * Get Ranks. Returns a 424 status code if the model is not a Sequence Classification model with + * @description Get Ranks. Returns a 424 status code if the model is not a Sequence Classification model with + * a single class. + */ + post: operations['rerank'] + } + '/tokenize': { + /** + * Tokenize inputs + * @description Tokenize inputs + */ + post: operations['tokenize'] + } + '/vertex': { + /** + * Generate embeddings from a Vertex request + * @description Generate embeddings from a Vertex request + */ + post: operations['vertex_compatibility'] + } +} + +export type webhooks = Record + +export interface components { + schemas: { + ClassifierModel: { + /** + * @example { + * "0": "LABEL" + * } + */ + id2label: { + [key: string]: string + } + /** + * @example { + * "LABEL": 0 + * } + */ + label2id: { + [key: string]: number + } + } + DecodeRequest: { + ids: components['schemas']['InputIds'] + /** + * @default true + * @example true + */ + skip_special_tokens?: boolean + } + /** + * @example [ + * "test" + * ] + */ + DecodeResponse: string[] + EmbedAllRequest: { + inputs: components['schemas']['Input'] + /** + * @default false + * @example false + */ + truncate?: boolean + } + /** + * @example [ + * [ + * [ + * 0, + * 1, + * 2 + * ] + * ] + * ] + */ + EmbedAllResponse: number[][][] + EmbedRequest: { + inputs: components['schemas']['Input'] + /** + * @default true + * @example true + */ + normalize?: boolean + /** + * @default false + * @example false + */ + truncate?: boolean + } + /** + * @example [ + * [ + * 0, + * 1, + * 2 + * ] + * ] + */ + EmbedResponse: number[][] + EmbedSparseRequest: { + inputs: components['schemas']['Input'] + /** + * @default false + * @example false + */ + truncate?: boolean + } + EmbedSparseResponse: components['schemas']['SparseValue'][][] + EmbeddingModel: { + /** @example cls */ + pooling: string + } + ErrorResponse: { + error: string + error_type: components['schemas']['ErrorType'] + } + /** @enum {string} */ + ErrorType: 'Unhealthy' | 'Backend' | 'Overloaded' | 'Validation' | 'Tokenizer' + Info: { + /** @example null */ + docker_label?: string | null + /** + * @default null + * @example null + */ + max_batch_requests?: number | null + /** @example 2048 */ + max_batch_tokens: number + /** @example 32 */ + max_client_batch_size: number + /** + * @description Router Parameters + * @example 128 + */ + max_concurrent_requests: number + /** @example 512 */ + max_input_length: number + /** @example float16 */ + model_dtype: string + /** + * @description Model info + * @example thenlper/gte-base + */ + model_id: string + /** @example fca14538aa9956a46526bd1d0d11d69e19b5a101 */ + model_sha?: string | null + model_type: components['schemas']['ModelType'] + /** @example null */ + sha?: string | null + /** @example 4 */ + tokenization_workers: number + /** + * @description Router Info + * @example 0.5.0 + */ + version: string + } + Input: string | string[] + InputIds: number[] | number[][] + ModelType: OneOf< + [ + { + classifier: components['schemas']['ClassifierModel'] + }, + { + embedding: components['schemas']['EmbeddingModel'] + }, + { + reranker: components['schemas']['ClassifierModel'] + } + ] + > + OpenAICompatEmbedding: { + /** + * @example [ + * 0, + * 1, + * 2 + * ] + */ + embedding: number[] + /** @example 0 */ + index: number + /** @example embedding */ + object: string + } + OpenAICompatErrorResponse: { + /** Format: int32 */ + code: number + error_type: components['schemas']['ErrorType'] + message: string + } + OpenAICompatRequest: { + input: components['schemas']['Input'] + /** @example null */ + model?: string | null + /** @example null */ + user?: string | null + } + OpenAICompatResponse: { + data: components['schemas']['OpenAICompatEmbedding'][] + /** @example thenlper/gte-base */ + model: string + /** @example list */ + object: string + usage: components['schemas']['OpenAICompatUsage'] + } + OpenAICompatUsage: { + /** @example 512 */ + prompt_tokens: number + /** @example 512 */ + total_tokens: number + } + /** + * @description Model input. Can be either a single string, a pair of strings or a batch of mixed single and pairs of strings. + * @example What is Deep Learning? + */ + PredictInput: string | string[] | string[][] + PredictRequest: { + inputs: components['schemas']['PredictInput'] + /** + * @default false + * @example false + */ + raw_scores?: boolean + /** + * @default false + * @example false + */ + truncate?: boolean + } + PredictResponse: components['schemas']['Prediction'][] | components['schemas']['Prediction'][][] + Prediction: { + /** @example admiration */ + label: string + /** + * Format: float + * @example 0.5 + */ + score: number + } + Rank: { + /** @example 0 */ + index: number + /** + * Format: float + * @example 1.0 + */ + score: number + /** + * @default null + * @example Deep Learning is ... + */ + text?: string | null + } + RerankRequest: { + /** @example What is Deep Learning? */ + query: string + /** + * @default false + * @example false + */ + raw_scores?: boolean + /** + * @default false + * @example false + */ + return_text?: boolean + /** + * @example [ + * "Deep Learning is ..." + * ] + */ + texts: string[] + /** + * @default false + * @example false + */ + truncate?: boolean + } + RerankResponse: components['schemas']['Rank'][] + SimpleToken: { + /** + * Format: int32 + * @example 0 + */ + id: number + /** @example false */ + special: boolean + /** @example 0 */ + start?: number | null + /** @example 2 */ + stop?: number | null + /** @example test */ + text: string + } + SparseValue: { + index: number + /** Format: float */ + value: number + } + TokenizeRequest: { + /** + * @default true + * @example true + */ + add_special_tokens?: boolean + inputs: components['schemas']['Input'] + } + /** + * @example [ + * [ + * { + * "id": 0, + * "special": false, + * "start": 0, + * "stop": 2, + * "text": "test" + * } + * ] + * ] + */ + TokenizeResponse: components['schemas']['SimpleToken'][][] + VertexInstance: + | (components['schemas']['EmbedRequest'] & { + /** @enum {string} */ + type: 'embed' + }) + | (components['schemas']['EmbedAllRequest'] & { + /** @enum {string} */ + type: 'embed_all' + }) + | (components['schemas']['EmbedSparseRequest'] & { + /** @enum {string} */ + type: 'embed_sparse' + }) + | (components['schemas']['PredictRequest'] & { + /** @enum {string} */ + type: 'predict' + }) + | (components['schemas']['RerankRequest'] & { + /** @enum {string} */ + type: 'rerank' + }) + | (components['schemas']['TokenizeRequest'] & { + /** @enum {string} */ + type: 'tokenize' + }) + VertexRequest: { + instances: components['schemas']['VertexInstance'][] + } + VertexResponse: components['schemas']['VertexResponseInstance'][] + VertexResponseInstance: + | { + result: components['schemas']['EmbedResponse'] + /** @enum {string} */ + type: 'embed' + } + | { + result: components['schemas']['EmbedAllResponse'] + /** @enum {string} */ + type: 'embed_all' + } + | { + result: components['schemas']['EmbedSparseResponse'] + /** @enum {string} */ + type: 'embed_sparse' + } + | { + result: components['schemas']['PredictResponse'] + /** @enum {string} */ + type: 'predict' + } + | { + result: components['schemas']['RerankResponse'] + /** @enum {string} */ + type: 'rerank' + } + | { + result: components['schemas']['TokenizeResponse'] + /** @enum {string} */ + type: 'tokenize' + } + } + responses: never + parameters: never + requestBodies: never + headers: never + pathItems: never +} + +export type $defs = Record + +export type external = Record + +export interface operations { + /** + * Decode input ids + * @description Decode input ids + */ + decode: { + requestBody: { + content: { + 'application/json': components['schemas']['DecodeRequest'] + } + } + responses: { + /** @description Decoded ids */ + 200: { + content: { + 'application/json': components['schemas']['DecodeResponse'] + } + } + /** @description Tokenization error */ + 422: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + } + } + /** + * Get Embeddings. Returns a 424 status code if the model is not an embedding model. + * @description Get Embeddings. Returns a 424 status code if the model is not an embedding model. + */ + embed: { + requestBody: { + content: { + 'application/json': components['schemas']['EmbedRequest'] + } + } + responses: { + /** @description Embeddings */ + 200: { + content: { + 'application/json': components['schemas']['EmbedResponse'] + } + } + /** @description Batch size error */ + 413: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Tokenization error */ + 422: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Embedding Error */ + 424: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Model is overloaded */ + 429: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + } + } + /** + * Get all Embeddings without Pooling. + * @description Get all Embeddings without Pooling. + * Returns a 424 status code if the model is not an embedding model. + */ + embed_all: { + requestBody: { + content: { + 'application/json': components['schemas']['EmbedAllRequest'] + } + } + responses: { + /** @description Embeddings */ + 200: { + content: { + 'application/json': components['schemas']['EmbedAllResponse'] + } + } + /** @description Batch size error */ + 413: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Tokenization error */ + 422: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Embedding Error */ + 424: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Model is overloaded */ + 429: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + } + } + /** + * Get Sparse Embeddings. Returns a 424 status code if the model is not an embedding model with SPLADE pooling. + * @description Get Sparse Embeddings. Returns a 424 status code if the model is not an embedding model with SPLADE pooling. + */ + embed_sparse: { + requestBody: { + content: { + 'application/json': components['schemas']['EmbedSparseRequest'] + } + } + responses: { + /** @description Embeddings */ + 200: { + content: { + 'application/json': components['schemas']['EmbedSparseResponse'] + } + } + /** @description Batch size error */ + 413: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Tokenization error */ + 422: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Embedding Error */ + 424: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Model is overloaded */ + 429: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + } + } + /** + * OpenAI compatible route. Returns a 424 status code if the model is not an embedding model. + * @description OpenAI compatible route. Returns a 424 status code if the model is not an embedding model. + */ + openai_embed: { + requestBody: { + content: { + 'application/json': components['schemas']['OpenAICompatRequest'] + } + } + responses: { + /** @description Embeddings */ + 200: { + content: { + 'application/json': components['schemas']['OpenAICompatResponse'] + } + } + /** @description Batch size error */ + 413: { + content: { + 'application/json': components['schemas']['OpenAICompatErrorResponse'] + } + } + /** @description Tokenization error */ + 422: { + content: { + 'application/json': components['schemas']['OpenAICompatErrorResponse'] + } + } + /** @description Embedding Error */ + 424: { + content: { + 'application/json': components['schemas']['OpenAICompatErrorResponse'] + } + } + /** @description Model is overloaded */ + 429: { + content: { + 'application/json': components['schemas']['OpenAICompatErrorResponse'] + } + } + } + } + /** + * Health check method + * @description Health check method + */ + health: { + responses: { + /** @description Everything is working fine */ + 200: { + content: never + } + /** @description Text embeddings Inference is down */ + 503: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + } + } + /** + * Text Embeddings Inference endpoint info + * @description Text Embeddings Inference endpoint info + */ + get_model_info: { + responses: { + /** @description Served model info */ + 200: { + content: { + 'application/json': components['schemas']['Info'] + } + } + } + } + /** + * Prometheus metrics scrape endpoint + * @description Prometheus metrics scrape endpoint + */ + metrics: { + responses: { + /** @description Prometheus Metrics */ + 200: { + content: { + 'text/plain': string + } + } + } + } + /** + * Get Predictions. Returns a 424 status code if the model is not a Sequence Classification model + * @description Get Predictions. Returns a 424 status code if the model is not a Sequence Classification model + */ + predict: { + requestBody: { + content: { + 'application/json': components['schemas']['PredictRequest'] + } + } + responses: { + /** @description Predictions */ + 200: { + content: { + 'application/json': components['schemas']['PredictResponse'] + } + } + /** @description Batch size error */ + 413: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Tokenization error */ + 422: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Prediction Error */ + 424: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Model is overloaded */ + 429: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + } + } + /** + * Get Ranks. Returns a 424 status code if the model is not a Sequence Classification model with + * @description Get Ranks. Returns a 424 status code if the model is not a Sequence Classification model with + * a single class. + */ + rerank: { + requestBody: { + content: { + 'application/json': components['schemas']['RerankRequest'] + } + } + responses: { + /** @description Ranks */ + 200: { + content: { + 'application/json': components['schemas']['RerankResponse'] + } + } + /** @description Batch size error */ + 413: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Tokenization error */ + 422: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Rerank Error */ + 424: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Model is overloaded */ + 429: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + } + } + /** + * Tokenize inputs + * @description Tokenize inputs + */ + tokenize: { + requestBody: { + content: { + 'application/json': components['schemas']['TokenizeRequest'] + } + } + responses: { + /** @description Tokenized ids */ + 200: { + content: { + 'application/json': components['schemas']['TokenizeResponse'] + } + } + /** @description Tokenization error */ + 422: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + } + } + /** + * Generate embeddings from a Vertex request + * @description Generate embeddings from a Vertex request + */ + vertex_compatibility: { + requestBody: { + content: { + 'application/json': components['schemas']['VertexRequest'] + } + } + responses: { + /** @description Results */ + 200: { + content: never + } + /** @description Batch size error */ + 413: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Tokenization error */ + 422: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Error */ + 424: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + /** @description Model is overloaded */ + 429: { + content: { + 'application/json': components['schemas']['ErrorResponse'] + } + } + } + } +} diff --git a/packages/embedder/tsconfig.json b/packages/embedder/tsconfig.json index 1d179d08096..b75106d1a8a 100644 --- a/packages/embedder/tsconfig.json +++ b/packages/embedder/tsconfig.json @@ -3,13 +3,10 @@ "compilerOptions": { "baseUrl": "../", "paths": { - // when we import from lib, make goto-definition point to the src "parabol-server/*": ["server/*"], - "parabol-client/*": ["client/*"] + "parabol-client/*": ["client/*"], + "~/*": ["client/*"] }, - "outDir": "lib", - "lib": ["esnext"], - "types": ["node"] - }, - "files": ["../server/types/modules.d.ts"] + "lib": ["esnext"] + } } diff --git a/packages/embedder/types/modules.d.ts b/packages/embedder/types/modules.d.ts new file mode 100644 index 00000000000..276eb185fa4 --- /dev/null +++ b/packages/embedder/types/modules.d.ts @@ -0,0 +1,4 @@ +declare module 'franc-min' { + import f from 'franc' + export = f +} diff --git a/packages/embedder/types/shared.d.ts b/packages/embedder/types/shared.d.ts new file mode 100644 index 00000000000..e94d0d14dea --- /dev/null +++ b/packages/embedder/types/shared.d.ts @@ -0,0 +1 @@ +import '../../server/types/modules' diff --git a/packages/gql-executor/RedisStream.ts b/packages/gql-executor/RedisStream.ts index 739480bd51d..22798509396 100644 --- a/packages/gql-executor/RedisStream.ts +++ b/packages/gql-executor/RedisStream.ts @@ -2,24 +2,25 @@ import RedisInstance from 'parabol-server/utils/RedisInstance' type MessageValue = [prop: string, stringifiedData: string] type Message = [messageId: string, value: MessageValue] -type XReadGroupRes = [streamName: string, messages: Message[]] -export default class RedisStream implements AsyncIterableIterator { +type XReadGroupRes = [streamName: string, messages: [Message, ...Message[]]] +export default class RedisStream implements AsyncIterableIterator { private stream: string private consumerGroup: string // xreadgroup blocks until a response is received, so this needs its own connection - private redis = new RedisInstance('gql_stream') + private redis: RedisInstance private consumer: string constructor(stream: string, consumerGroup: string, consumer: string) { this.stream = stream this.consumerGroup = consumerGroup this.consumer = consumer + this.redis = new RedisInstance(stream) } [Symbol.asyncIterator]() { return this } - async next() { + async next(): Promise> { const response = await this.redis.xreadgroup( 'GROUP', this.consumerGroup, diff --git a/packages/gql-executor/gqlExecutor.ts b/packages/gql-executor/gqlExecutor.ts index 2d4239b22a6..a3ce88d0853 100644 --- a/packages/gql-executor/gqlExecutor.ts +++ b/packages/gql-executor/gqlExecutor.ts @@ -2,9 +2,10 @@ import tracer from 'dd-trace' import {ServerChannel} from 'parabol-client/types/constEnums' import GQLExecutorChannelId from '../client/shared/gqlIds/GQLExecutorChannelId' import SocketServerChannelId from '../client/shared/gqlIds/SocketServerChannelId' -import executeGraphQL, {GQLRequest} from '../server/graphql/executeGraphQL' +import executeGraphQL from '../server/graphql/executeGraphQL' import '../server/initSentry' import '../server/monkeyPatchFetch' +import {GQLRequest} from '../server/types/custom' import RedisInstance from '../server/utils/RedisInstance' import RedisStream from './RedisStream' @@ -16,7 +17,7 @@ tracer.init({ }) tracer.use('ioredis').use('http').use('pg') -const {REDIS_URL, SERVER_ID} = process.env +const {SERVER_ID} = process.env interface PubSubPromiseMessage { jobId: string socketServerId: string @@ -26,7 +27,7 @@ interface PubSubPromiseMessage { const run = async () => { const publisher = new RedisInstance('gql_pub') const subscriber = new RedisInstance('gql_sub') - const executorChannel = GQLExecutorChannelId.join(SERVER_ID) + const executorChannel = GQLExecutorChannelId.join(SERVER_ID!) // on shutdown, remove consumer from the group process.on('SIGTERM', async () => { diff --git a/packages/gql-executor/modules.d.ts b/packages/gql-executor/modules.d.ts new file mode 100644 index 00000000000..8a2be20ba0c --- /dev/null +++ b/packages/gql-executor/modules.d.ts @@ -0,0 +1,2 @@ +import '../server/types/modules' +import '../server/types/webpackEnv' diff --git a/packages/gql-executor/package.json b/packages/gql-executor/package.json index bed8c3473eb..3797dee3e75 100644 --- a/packages/gql-executor/package.json +++ b/packages/gql-executor/package.json @@ -1,6 +1,6 @@ { "name": "gql-executor", - "version": "7.23.1", + "version": "7.24.1", "description": "A Stateless GraphQL Executor", "author": "Matt Krick ", "homepage": "https://github.com/ParabolInc/parabol/tree/master/packages/gqlExecutor#readme", @@ -27,8 +27,8 @@ }, "dependencies": { "dd-trace": "^4.2.0", - "parabol-client": "7.23.1", - "parabol-server": "7.23.1", + "parabol-client": "7.24.1", + "parabol-server": "7.24.1", "undici": "^5.26.2" } } diff --git a/packages/gql-executor/tsconfig.json b/packages/gql-executor/tsconfig.json index 1d179d08096..5f1fdbf2d75 100644 --- a/packages/gql-executor/tsconfig.json +++ b/packages/gql-executor/tsconfig.json @@ -1,15 +1,14 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { + "esModuleInterop": true, "baseUrl": "../", "paths": { // when we import from lib, make goto-definition point to the src "parabol-server/*": ["server/*"], - "parabol-client/*": ["client/*"] + "parabol-client/*": ["client/*"], + "~/*": ["client/*"] }, - "outDir": "lib", - "lib": ["esnext"], - "types": ["node"] - }, - "files": ["../server/types/modules.d.ts"] + "lib": ["esnext"] + } } diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json index 2cbc84391c1..84ae4bb4e29 100644 --- a/packages/integration-tests/package.json +++ b/packages/integration-tests/package.json @@ -2,7 +2,7 @@ "name": "integration-tests", "author": "Parabol Inc. (http://github.com/ParabolInc)", "license": "AGPL-3.0", - "version": "7.23.1", + "version": "7.24.1", "description": "", "main": "index.js", "scripts": { @@ -11,8 +11,6 @@ }, "devDependencies": { "@playwright/test": "^1.34.3", - "eslint": "^8.8.0", - "eslint-config-prettier": "^8.5.0", "lint-staged": "^12.3.3", "ts-app-env": "^1.4.2", "typescript": "^5.3.3" diff --git a/packages/server/__tests__/autoJoin.test.ts b/packages/server/__tests__/autoJoin.test.ts index 85bc9e3e222..3fe4564e6eb 100644 --- a/packages/server/__tests__/autoJoin.test.ts +++ b/packages/server/__tests__/autoJoin.test.ts @@ -1,11 +1,11 @@ import faker from 'faker' -import {getUserTeams, sendPublic, sendIntranet, signUp, signUpWithEmail} from './common' import getRethink from '../database/rethinkDriver' import createEmailVerification from '../email/createEmailVerification' +import {getUserTeams, sendIntranet, sendPublic} from './common' const signUpVerified = async (email: string) => { const password = faker.internet.password() - const signUp = await sendPublic({ + await sendPublic({ query: ` mutation SignUpWithPassword($email: ID!, $password: String!) { signUpWithPassword(email: $email, password: $password, params: "") { diff --git a/packages/server/__tests__/common.ts b/packages/server/__tests__/common.ts index f9ca34ab293..93c134b229b 100644 --- a/packages/server/__tests__/common.ts +++ b/packages/server/__tests__/common.ts @@ -199,5 +199,5 @@ export const getUserTeams = async (userId: string) => { } } }) - return user.data.user.teams + return user.data.user.teams as [{id: string}, ...{id: string}[]] } diff --git a/packages/server/__tests__/disableAnonymity.test.ts b/packages/server/__tests__/disableAnonymity.test.ts index 7143981aa36..1887c3357a0 100644 --- a/packages/server/__tests__/disableAnonymity.test.ts +++ b/packages/server/__tests__/disableAnonymity.test.ts @@ -89,8 +89,12 @@ const startRetro = async (teamId: string, authToken: string) => { }, authToken }) - - const meeting = startRetroQuery.data.startRetrospective.meeting + const meeting = startRetroQuery.data.startRetrospective.meeting as { + id: string + phases: { + reflectPrompts: [{id: string}] + }[] + } return meeting } @@ -134,7 +138,7 @@ test('By default all reflections are anonymous', async () => { expect(meetingSettings.disableAnonymity).toEqual(false) const meeting = await startRetro(teamId, authToken) - const reflectPrompts = meeting.phases.find(({reflectPrompts}) => !!reflectPrompts).reflectPrompts + const reflectPrompts = meeting.phases.find(({reflectPrompts}) => !!reflectPrompts)!.reflectPrompts const reflection = await addReflection(meeting.id, reflectPrompts[0].id, authToken) expect(reflection).toEqual({ @@ -153,7 +157,7 @@ test('Creator is visible when disableAnonymity is set', async () => { expect(updatedMeetingSettings.disableAnonymity).toEqual(true) const meeting = await startRetro(teamId, authToken) - const reflectPrompts = meeting.phases.find(({reflectPrompts}) => !!reflectPrompts).reflectPrompts + const reflectPrompts = meeting.phases.find(({reflectPrompts}) => !!reflectPrompts)!.reflectPrompts const reflection = await addReflection(meeting.id, reflectPrompts[0].id, authToken) expect(reflection).toEqual({ @@ -172,7 +176,7 @@ test('Super user can always read creatorId of a reflection', async () => { expect(meetingSettings.disableAnonymity).toEqual(false) const meeting = await startRetro(teamId, authToken) - const reflectPrompts = meeting.phases.find(({reflectPrompts}) => !!reflectPrompts).reflectPrompts + const reflectPrompts = meeting.phases.find(({reflectPrompts}) => !!reflectPrompts)!.reflectPrompts await addReflection(meeting.id, reflectPrompts[0].id, authToken) diff --git a/packages/server/__tests__/loginSAML.test.ts b/packages/server/__tests__/loginSAML.test.ts index 426f9606140..312bb3154e5 100644 --- a/packages/server/__tests__/loginSAML.test.ts +++ b/packages/server/__tests__/loginSAML.test.ts @@ -10,30 +10,6 @@ test.skip('SAML', async () => { const orgId = `${samlName}-orgId` const domain = 'example.com' - const _metadata = ` - - - - - - - MIICUDCCAbmgAwIBAgIBADANBgkqhkiG9w0BAQ0FADBFMQswCQYDVQQGEwJ1czELMAkGA1UECAwCQ0ExEzARBgNVBAoMCkV4YW1wbGUgQ28xFDASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTIxMDkxMDA3NTkzMFoXDTI2MDkwOTA3NTkzMFowRTELMAkGA1UEBhMCdXMxCzAJBgNVBAgMAkNBMRMwEQYDVQQKDApFeGFtcGxlIENvMRQwEgYDVQQDDAtleGFtcGxlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEArg9DZwR9v7Vok1IW+hIpYin9llPBh1MV5CxjfK596EwuadyQuko3jGv8qDlx4tG6JiGTjQfCuzJVAhYi2OKuKBqyJewKoen1uF0dRyws9n6zZl0GsVJkObdrNo5P6eib3VOsXPJ10RjxWsWx5WRur2dYdkOJFxC6zN1IbXSXYYMCAwEAAaNQME4wHQYDVR0OBBYEFKr/1y4R+kamPz623HnHM7tz6C4XMB8GA1UdIwQYMBaAFKr/1y4R+kamPz623HnHM7tz6C4XMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQENBQADgYEALKBl6QPk9HMB5V+GYu50XNFmzyuuXt3zAKMSYcyhxVSBCe6SKw1iqvvPza4rGp7DpeJI/8R3qBTuZqfl0rX624wvHGc4N9WubMLPejAn7dMu3oGfm9KUX+Um1RG0U6zsi9t3X90rroea/5SQvw/uAWUxS59U2r8massI/WFJKh8= - - - - - - - MIICUDCCAbmgAwIBAgIBADANBgkqhkiG9w0BAQ0FADBFMQswCQYDVQQGEwJ1czELMAkGA1UECAwCQ0ExEzARBgNVBAoMCkV4YW1wbGUgQ28xFDASBgNVBAMMC2V4YW1wbGUuY29tMB4XDTIxMDkxMDA3NTkzMFoXDTI2MDkwOTA3NTkzMFowRTELMAkGA1UEBhMCdXMxCzAJBgNVBAgMAkNBMRMwEQYDVQQKDApFeGFtcGxlIENvMRQwEgYDVQQDDAtleGFtcGxlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEArg9DZwR9v7Vok1IW+hIpYin9llPBh1MV5CxjfK596EwuadyQuko3jGv8qDlx4tG6JiGTjQfCuzJVAhYi2OKuKBqyJewKoen1uF0dRyws9n6zZl0GsVJkObdrNo5P6eib3VOsXPJ10RjxWsWx5WRur2dYdkOJFxC6zN1IbXSXYYMCAwEAAaNQME4wHQYDVR0OBBYEFKr/1y4R+kamPz623HnHM7tz6C4XMB8GA1UdIwQYMBaAFKr/1y4R+kamPz623HnHM7tz6C4XMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQENBQADgYEALKBl6QPk9HMB5V+GYu50XNFmzyuuXt3zAKMSYcyhxVSBCe6SKw1iqvvPza4rGp7DpeJI/8R3qBTuZqfl0rX624wvHGc4N9WubMLPejAn7dMu3oGfm9KUX+Um1RG0U6zsi9t3X90rroea/5SQvw/uAWUxS59U2r8massI/WFJKh8= - - - - urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified - - - - ` - const verifyDomain = await sendIntranet({ query: ` mutation VerifyDomain($slug: ID!, $addDomains: [ID!], $orgId: ID!) { diff --git a/packages/server/__tests__/processRecurrence.test.ts b/packages/server/__tests__/processRecurrence.test.ts index f6e6c685715..fab6970bc9a 100644 --- a/packages/server/__tests__/processRecurrence.test.ts +++ b/packages/server/__tests__/processRecurrence.test.ts @@ -1,14 +1,13 @@ import ms from 'ms' import {RRule} from 'rrule' import getRethink from '../database/rethinkDriver' -import MeetingTeamPrompt from '../database/types/MeetingTeamPrompt' -import TeamPromptResponsesPhase from '../database/types/TeamPromptResponsesPhase' import MeetingRetrospective from '../database/types/MeetingRetrospective' +import MeetingTeamPrompt from '../database/types/MeetingTeamPrompt' import ReflectPhase from '../database/types/ReflectPhase' +import TeamPromptResponsesPhase from '../database/types/TeamPromptResponsesPhase' import generateUID from '../generateUID' import {insertMeetingSeries as insertMeetingSeriesQuery} from '../postgres/queries/insertMeetingSeries' import {getUserTeams, sendIntranet, signUp} from './common' -import createNewMeetingPhases from '../graphql/mutations/helpers/createNewMeetingPhases' const PROCESS_RECURRENCE = ` mutation { @@ -292,7 +291,11 @@ test('Should end the current retro meeting and start a new meeting', async () => facilitatorUserId: userId, scheduledEndTime: new Date(Date.now() - ms('5m')), meetingSeriesId, - templateId: 'startStopContinueTemplate' + templateId: 'startStopContinueTemplate', + disableAnonymity: false, + totalVotes: 5, + name: '', + maxVotesPerGroup: 5 }) // The last meeting in the series was created just over 24h ago, so the next one should start diff --git a/packages/server/__tests__/startRetrospective.test.ts b/packages/server/__tests__/startRetrospective.test.ts index 22ff21a7f12..17deae321e5 100644 --- a/packages/server/__tests__/startRetrospective.test.ts +++ b/packages/server/__tests__/startRetrospective.test.ts @@ -1,14 +1,8 @@ -import ms from 'ms' -import {RRule} from 'rrule' import getRethink from '../database/rethinkDriver' -import MeetingTeamPrompt from '../database/types/MeetingTeamPrompt' -import TeamPromptResponsesPhase from '../database/types/TeamPromptResponsesPhase' -import generateUID from '../generateUID' -import {insertMeetingSeries as insertMeetingSeriesQuery} from '../postgres/queries/insertMeetingSeries' -import {getUserTeams, sendIntranet, sendPublic, signUp} from './common' +import {getUserTeams, sendPublic, signUp} from './common' test('Retro is named Retro #1 by default', async () => { - const r = await getRethink() + await getRethink() const {userId, authToken} = await signUp() const {id: teamId} = (await getUserTeams(userId))[0] @@ -48,7 +42,7 @@ test('Retro is named Retro #1 by default', async () => { }) test('Recurring retro is named like RetroSeries Jan 1', async () => { - const r = await getRethink() + await getRethink() const {userId, authToken} = await signUp() const {id: teamId} = (await getUserTeams(userId))[0] @@ -81,7 +75,11 @@ test('Recurring retro is named like RetroSeries Jan 1', async () => { authToken }) - const formattedDate = now.toLocaleDateString('en-US', {month: 'short', day: 'numeric'}, 'UTC') + const formattedDate = now.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + timeZone: 'UTC' + }) expect(newRetro).toMatchObject({ data: { startRetrospective: { diff --git a/packages/server/database/rethinkDriver.ts b/packages/server/database/rethinkDriver.ts index 5f7311c0580..370482933b2 100644 --- a/packages/server/database/rethinkDriver.ts +++ b/packages/server/database/rethinkDriver.ts @@ -4,7 +4,7 @@ import SlackAuth from '../database/types/SlackAuth' import SlackNotification from '../database/types/SlackNotification' import TeamInvitation from '../database/types/TeamInvitation' import TeamMember from '../database/types/TeamMember' -import {ScheduledJobUnion} from '../graphql/private/mutations/runScheduledJobs' +import {ScheduledJobUnion} from '../types/custom' import {AnyMeeting, AnyMeetingSettings, AnyMeetingTeamMember} from '../postgres/types/Meeting' import getRethinkConfig from './getRethinkConfig' import {R} from './stricterR' diff --git a/packages/server/dataloader/__tests__/isCompanyDomain.test.ts b/packages/server/dataloader/__tests__/isCompanyDomain.test.ts index 172958a60d6..d364d2a7d61 100644 --- a/packages/server/dataloader/__tests__/isCompanyDomain.test.ts +++ b/packages/server/dataloader/__tests__/isCompanyDomain.test.ts @@ -1,7 +1,5 @@ -import faker from 'faker' import '../../../../scripts/webpack/utils/dotenv' import getDataLoader from '../../graphql/getDataLoader' -import getPg from '../../postgres/getPg' const dataloader = getDataLoader() diff --git a/packages/server/dataloader/__tests__/isOrgVerified.test.ts b/packages/server/dataloader/__tests__/isOrgVerified.test.ts index b12897d4b3a..58247c515ab 100644 --- a/packages/server/dataloader/__tests__/isOrgVerified.test.ts +++ b/packages/server/dataloader/__tests__/isOrgVerified.test.ts @@ -1,18 +1,22 @@ /* eslint-env jest */ -import {MasterPool, r} from 'rethinkdb-ts' -import getRedis from '../../utils/getRedis' +import {r} from 'rethinkdb-ts' import getRethinkConfig from '../../database/getRethinkConfig' import getRethink from '../../database/rethinkDriver' -import isUserVerified from '../../utils/isUserVerified' +import OrganizationUser from '../../database/types/OrganizationUser' import generateUID from '../../generateUID' +import {DataLoaderWorker} from '../../graphql/graphql' +import getRedis from '../../utils/getRedis' +import isUserVerified from '../../utils/isUserVerified' +import RootDataLoader from '../RootDataLoader' import {isOrgVerified} from '../customLoaderMakers' jest.mock('../../database/rethinkDriver') jest.mock('../../utils/isUserVerified') -getRethink.mockImplementation(() => { - return r +jest.mocked(getRethink).mockImplementation(() => { + return r as any }) -isUserVerified.mockImplementation(() => { + +jest.mocked(isUserVerified).mockImplementation(() => { return true }) @@ -24,7 +28,7 @@ const testConfig = { db: TEST_DB } -const createTables = async (...tables: string) => { +const createTables = async (...tables: string[]) => { for (const tableName of tables) { const structure = await r .db('rethinkdb') @@ -40,9 +44,10 @@ const createTables = async (...tables: string) => { } } -type TestOrganizationUser = Pick< - OrganizationUser, - 'inactive' | 'joinedAt' | 'removedAt' | 'role' | 'userId' +type TestOrganizationUser = Partial< + Pick & { + domain: string + } > const userLoader = { @@ -61,9 +66,9 @@ const dataLoader = { users: userLoader, isCompanyDomain: isCompanyDomainLoader } - return loaders[loader] + return loaders[loader as keyof typeof loaders] }) -} +} as any as DataLoaderWorker const addOrg = async ( activeDomain: string | null, @@ -101,10 +106,10 @@ const addOrg = async ( return orgId } -const isOrgVerifiedLoader = isOrgVerified(dataLoader) +const isOrgVerifiedLoader = isOrgVerified(dataLoader as any as RootDataLoader) beforeAll(async () => { - const conn = await r.connectPool(testConfig) + await r.connectPool(testConfig) try { await r.dbDrop(TEST_DB).run() } catch (e) { @@ -121,7 +126,7 @@ afterEach(async () => { }) afterAll(async () => { - await r.getPoolMaster().drain() + await r.getPoolMaster()?.drain() getRedis().quit() }) @@ -203,12 +208,12 @@ test('Empty org does not throw', async () => { }) test('Orgs with verified emails from different domains do not qualify', async () => { - const org1 = await addOrg('parabol.co', [ + await addOrg('parabol.co', [ { joinedAt: new Date('2023-09-06'), userId: 'founder1', domain: 'not-parabol.co' - } + } as any ]) const isVerified = await isOrgVerifiedLoader.load('parabol.co') diff --git a/packages/server/dataloader/__tests__/usersCustomRedisQueries.test.ts b/packages/server/dataloader/__tests__/usersCustomRedisQueries.test.ts index 0aa1d0a3162..50fcc7fa77f 100644 --- a/packages/server/dataloader/__tests__/usersCustomRedisQueries.test.ts +++ b/packages/server/dataloader/__tests__/usersCustomRedisQueries.test.ts @@ -1,6 +1,7 @@ import faker from 'faker' import '../../../../scripts/webpack/utils/dotenv' import getDataLoader from '../../graphql/getDataLoader' +import isValid from '../../graphql/isValid' import getPg from '../../postgres/getPg' afterAll(async () => { @@ -17,16 +18,19 @@ test('Result is mapped to correct id', async () => { const dataloader = getDataLoader() const expectedUsers = faker.helpers.shuffle( - (await pg.query('SELECT "id", "email" FROM "User" LIMIT 100')).rows + (await pg.query('SELECT "id", "email" FROM "User" LIMIT 100')).rows as { + id: string + email: string + }[] ) const userIds = expectedUsers.map(({id}) => id) - const actualUsers = (await (dataloader.get('users') as any).loadMany(userIds)).map( - ({id, email}) => ({ + const actualUsers = (await dataloader.get('users').loadMany(userIds)) + .filter(isValid) + .map(({id, email}) => ({ id, email - }) - ) + })) console.log('Ran with #users:', actualUsers.length) diff --git a/packages/server/dataloader/customLoaderMakers.ts b/packages/server/dataloader/customLoaderMakers.ts index ac7e1ea6ec7..f10dfd1f257 100644 --- a/packages/server/dataloader/customLoaderMakers.ts +++ b/packages/server/dataloader/customLoaderMakers.ts @@ -5,11 +5,11 @@ import getRethink, {RethinkSchema} from '../database/rethinkDriver' import {RDatum} from '../database/stricterR' import MeetingSettingsTeamPrompt from '../database/types/MeetingSettingsTeamPrompt' import MeetingTemplate from '../database/types/MeetingTemplate' +import Organization from '../database/types/Organization' import OrganizationUser from '../database/types/OrganizationUser' import {Reactable, ReactableEnum} from '../database/types/Reactable' import Task, {TaskStatusEnum} from '../database/types/Task' import isValid from '../graphql/isValid' -import {Organization} from '../graphql/public/resolverTypes' import {SAMLSource} from '../graphql/public/types/SAML' import getKysely from '../postgres/getKysely' import {TeamMeetingTemplate} from '../postgres/pg.d' diff --git a/packages/server/dataloader/rethinkForeignKeyLoaderMakers.ts b/packages/server/dataloader/rethinkForeignKeyLoaderMakers.ts index 7c19ba2d254..70a038b712c 100644 --- a/packages/server/dataloader/rethinkForeignKeyLoaderMakers.ts +++ b/packages/server/dataloader/rethinkForeignKeyLoaderMakers.ts @@ -192,6 +192,19 @@ export const retroReflectionsByMeetingId = new RethinkForeignKeyLoaderMaker( } ) +export const retroReflectionsByGroupId = new RethinkForeignKeyLoaderMaker( + 'retroReflections', + 'reflectionGroupId', + async (reflectionGroupIds) => { + const r = await getRethink() + return r + .table('RetroReflection') + .getAll(r.args(reflectionGroupIds), {index: 'reflectionGroupId'}) + .filter({isActive: true}) + .run() + } +) + export const templateDimensionsByTemplateId = new RethinkForeignKeyLoaderMaker( 'templateDimensions', 'templateId', diff --git a/packages/server/email/inlineImages.ts b/packages/server/email/inlineImages.ts index d02c7bf6448..176a10d5798 100644 --- a/packages/server/email/inlineImages.ts +++ b/packages/server/email/inlineImages.ts @@ -20,7 +20,7 @@ const getFile = async (pathname: string) => { try { const res = await fetch(pathname) if (res.status !== 200) return null - data = await res.buffer() + data = await (res as any).buffer() } catch (e) { return null } @@ -42,7 +42,7 @@ const inlineImages = async (html: string) => { $('body') .find('img') .each((_i, img) => { - const pathname = $(img).attr('src') + const pathname = $(img).attr('src') as keyof typeof cidDict if (!pathname) return cidDict[pathname] = cidDict[pathname] || generateUID() + path.extname(pathname) $(img).attr('src', `cid:${cidDict[pathname]}`) @@ -51,7 +51,7 @@ const inlineImages = async (html: string) => { const files = await Promise.all(uniquePathnames.map(getFile)) const options = files.map((data, idx) => { if (!data) return null - const pathname = uniquePathnames[idx] + const pathname = uniquePathnames[idx] as keyof typeof cidDict const filename = cidDict[pathname] return {data, filename} }) diff --git a/packages/server/generateUID.ts b/packages/server/generateUID.ts index 15d3a616e70..822843fa52d 100644 --- a/packages/server/generateUID.ts +++ b/packages/server/generateUID.ts @@ -8,7 +8,7 @@ const SEQ_BIT_LEN = 12 const TS_OFFSET = BigInt(MACHINE_ID_BIT_LEN + SEQ_BIT_LEN) const MID_OFFSET = BigInt(SEQ_BIT_LEN) const BIG_ZERO = BigInt(0) -const MAX_SEQ = 2 ** SEQ_BIT_LEN - 1 +export const MAX_SEQ = 2 ** SEQ_BIT_LEN - 1 // if MID overflows, we will generate duplicate ids, throw instead if (MID < 0 || MID > 2 ** MACHINE_ID_BIT_LEN - 1) { diff --git a/packages/server/graphql/executeGraphQL.ts b/packages/server/graphql/executeGraphQL.ts index 6b6ad34ea72..8fea40bc2a5 100644 --- a/packages/server/graphql/executeGraphQL.ts +++ b/packages/server/graphql/executeGraphQL.ts @@ -7,30 +7,13 @@ import tracer from 'dd-trace' import {graphql} from 'graphql' import {FormattedExecutionResult} from 'graphql/execution/execute' -import AuthToken from '../database/types/AuthToken' +import type {GQLRequest} from '../types/custom' import CompiledQueryCache from './CompiledQueryCache' import getDataLoader from './getDataLoader' import getRateLimiter from './getRateLimiter' import privateSchema from './private/rootSchema' import publicSchema from './public/rootSchema' -export interface GQLRequest { - authToken: AuthToken - ip?: string - socketId?: string - variables?: {[key: string]: any} - docId?: string - query?: string - rootValue?: {[key: string]: any} - dataLoaderId?: string - // true if the query is on the private schema - isPrivate?: boolean - // true if the query is ad-hoc (e.g. GraphiQL, CLI) - isAdHoc?: boolean - // Datadog opentracing span of the calling server - carrier?: any -} - const queryCache = new CompiledQueryCache() const executeGraphQL = async (req: GQLRequest) => { diff --git a/packages/server/graphql/mutations/helpers/publishToEmbedder.ts b/packages/server/graphql/mutations/helpers/publishToEmbedder.ts new file mode 100644 index 00000000000..c8a735f4ac9 --- /dev/null +++ b/packages/server/graphql/mutations/helpers/publishToEmbedder.ts @@ -0,0 +1,14 @@ +import type {MessageToEmbedder} from 'embedder/custom' +import getRedis from '../../../utils/getRedis' + +export const publishToEmbedder = (message: MessageToEmbedder) => { + return getRedis().xadd( + 'embedMetadataStream', + 'MAXLEN', + '~', + 1000, + '*', + 'msg', + JSON.stringify(message) + ) +} diff --git a/packages/server/graphql/mutations/helpers/safeEndRetrospective.ts b/packages/server/graphql/mutations/helpers/safeEndRetrospective.ts index cd116c65ea8..f5a69be4f22 100644 --- a/packages/server/graphql/mutations/helpers/safeEndRetrospective.ts +++ b/packages/server/graphql/mutations/helpers/safeEndRetrospective.ts @@ -1,35 +1,36 @@ +import {RawDraftContentState} from 'draft-js' import {SubscriptionChannel} from 'parabol-client/types/constEnums' import {DISCUSS, PARABOL_AI_USER_ID} from 'parabol-client/utils/constants' import getMeetingPhase from 'parabol-client/utils/getMeetingPhase' import findStageById from 'parabol-client/utils/meetings/findStageById' -import {RawDraftContentState} from 'draft-js' import {checkTeamsLimit} from '../../../billing/helpers/teamLimitsCheck' import getRethink from '../../../database/rethinkDriver' import {RDatum} from '../../../database/stricterR' import MeetingRetrospective from '../../../database/types/MeetingRetrospective' +import NotificationMentioned from '../../../database/types/NotificationMentioned' import TimelineEventRetroComplete from '../../../database/types/TimelineEventRetroComplete' import getKysely from '../../../postgres/getKysely' import removeSuggestedAction from '../../../safeMutations/removeSuggestedAction' +import {Logger} from '../../../utils/Logger' +import RecallAIServerManager from '../../../utils/RecallAIServerManager' import {analytics} from '../../../utils/analytics/analytics' import {getUserId} from '../../../utils/authorization' import getPhase from '../../../utils/getPhase' import publish from '../../../utils/publish' -import publishNotification from '../../public/mutations/helpers/publishNotification' -import RecallAIServerManager from '../../../utils/RecallAIServerManager' import sendToSentry from '../../../utils/sendToSentry' import standardError from '../../../utils/standardError' import {InternalContext} from '../../graphql' -import updateTeamInsights from './updateTeamInsights' +import publishNotification from '../../public/mutations/helpers/publishNotification' import sendNewMeetingSummary from './endMeeting/sendNewMeetingSummary' +import gatherInsights from './gatherInsights' import generateWholeMeetingSentimentScore from './generateWholeMeetingSentimentScore' import generateWholeMeetingSummary from './generateWholeMeetingSummary' import handleCompletedStage from './handleCompletedStage' import {IntegrationNotifier} from './notifications/IntegrationNotifier' +import {publishToEmbedder} from './publishToEmbedder' import removeEmptyTasks from './removeEmptyTasks' import updateQualAIMeetingsCount from './updateQualAIMeetingsCount' -import gatherInsights from './gatherInsights' -import NotificationMentioned from '../../../database/types/NotificationMentioned' -import {Logger} from '../../../utils/Logger' +import updateTeamInsights from './updateTeamInsights' const getTranscription = async (recallBotId?: string | null) => { if (!recallBotId) return @@ -370,6 +371,7 @@ const safeEndRetrospective = async ({ removedTaskIds, timelineEventId } + publishToEmbedder({objectTypes: ['retrospectiveDiscussionTopic'], meetingId}) publish(SubscriptionChannel.TEAM, teamId, 'EndRetrospectiveSuccess', data, subOptions) return data diff --git a/packages/server/graphql/private/mutations/runScheduledJobs.ts b/packages/server/graphql/private/mutations/runScheduledJobs.ts index aebb1e5eafe..5317aff0147 100644 --- a/packages/server/graphql/private/mutations/runScheduledJobs.ts +++ b/packages/server/graphql/private/mutations/runScheduledJobs.ts @@ -40,8 +40,6 @@ const processMeetingStageTimeLimits = async ( }) } -export type ScheduledJobUnion = ScheduledJobMeetingStageTimeLimit | ScheduledTeamLimitsJob - const processJob = async (job: Selectable, dataLoader: DataLoaderWorker) => { const pg = getKysely() const res = await pg.deleteFrom('ScheduledJob').where('id', '=', job.id).executeTakeFirst() diff --git a/packages/server/graphql/public/rootSchema.ts b/packages/server/graphql/public/rootSchema.ts index b6f1d26fad2..e65b1459189 100644 --- a/packages/server/graphql/public/rootSchema.ts +++ b/packages/server/graphql/public/rootSchema.ts @@ -99,5 +99,4 @@ const addRequestors = (schema: GraphQLSchema) => { const rootSchema = addRequestors(resolveTypesForMutationPayloads(parabolWithNestedResolversSchema)) -export type RootSchema = typeof rootSchema export default rootSchema diff --git a/packages/server/integrations/gitlab/GitLabServerManager.ts b/packages/server/integrations/gitlab/GitLabServerManager.ts index 2ffcfa826bf..dd44a3a5525 100644 --- a/packages/server/integrations/gitlab/GitLabServerManager.ts +++ b/packages/server/integrations/gitlab/GitLabServerManager.ts @@ -9,8 +9,8 @@ import getIssue from '../../graphql/nestedSchema/GitLab/queries/getIssue.graphql import getProfile from '../../graphql/nestedSchema/GitLab/queries/getProfile.graphql' import getProjectIssues from '../../graphql/nestedSchema/GitLab/queries/getProjectIssues.graphql' import getProjects from '../../graphql/nestedSchema/GitLab/queries/getProjects.graphql' -import {RootSchema} from '../../graphql/public/rootSchema' import {IGetTeamMemberIntegrationAuthQueryResult} from '../../postgres/queries/generated/getTeamMemberIntegrationAuthQuery' +import {RootSchema} from '../../types/custom' import { CreateIssueMutation, CreateNoteMutation, diff --git a/packages/server/package.json b/packages/server/package.json index 68cfcfbb846..d619f4b0860 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -3,7 +3,7 @@ "description": "An open-source app for building smarter, more agile teams.", "author": "Parabol Inc. (http://github.com/ParabolInc)", "license": "AGPL-3.0", - "version": "7.23.1", + "version": "7.24.1", "repository": { "type": "git", "url": "https://github.com/ParabolInc/parabol" @@ -32,6 +32,7 @@ "@types/bcryptjs": "^2.4.2", "@types/cheerio": "^0.22.30", "@types/dotenv": "^6.1.1", + "@types/faker": "^5.5.9", "@types/graphql": "^14.5.0", "@types/html-minifier-terser": "5.1.2", "@types/jest": "^29.5.1", @@ -49,8 +50,6 @@ "clean-webpack-plugin": "^3.0.0", "copy-webpack-plugin": "7.0.0", "css-loader": "5.0.1", - "eslint": "^8.2.0", - "eslint-config-prettier": "^8.5.0", "faker": "^5.5.3", "file-loader": "6.2.0", "graphql-typed": "^0.4.1", @@ -124,7 +123,7 @@ "oauth-1.0a": "^2.2.6", "openai": "^4.24.1", "oy-vey": "^0.12.1", - "parabol-client": "7.23.1", + "parabol-client": "7.24.1", "pg": "^8.5.1", "react": "^17.0.2", "react-dom": "^17.0.2", diff --git a/packages/server/postgres/migrations/1703031300000_addEmbeddingTables.ts b/packages/server/postgres/migrations/1703031300000_addEmbeddingTables.ts index e5f6f813877..33718ee75a1 100644 --- a/packages/server/postgres/migrations/1703031300000_addEmbeddingTables.ts +++ b/packages/server/postgres/migrations/1703031300000_addEmbeddingTables.ts @@ -1,7 +1,7 @@ import {Client} from 'pg' -import getPgConfig from '../getPgConfig' import {r} from 'rethinkdb-ts' import connectRethinkDB from '../../database/connectRethinkDB' +import getPgConfig from '../getPgConfig' export async function up() { const client = new Client(getPgConfig()) @@ -78,8 +78,8 @@ export async function down() { EXECUTE 'DROP TABLE IF EXISTS "EmbeddingsJobQueue"'; EXECUTE 'DROP TABLE IF EXISTS "EmbeddingsMetadata"'; - EXECUTE 'DROP TYPE IF EXISTS "EmbeddingsStateEnum"'; - EXECUTE 'DROP TYPE IF EXISTS "EmbeddingsObjectTypeEnum"'; + EXECUTE 'DROP TYPE IF EXISTS "EmbeddingsStateEnum" CASCADE'; + EXECUTE 'DROP TYPE IF EXISTS "EmbeddingsObjectTypeEnum" CASCADE'; END $$; `) await client.end() diff --git a/packages/server/postgres/migrations/1709934935000_embeddingsMetadataId.ts b/packages/server/postgres/migrations/1709934935000_embeddingsMetadataId.ts new file mode 100644 index 00000000000..185b1e0881b --- /dev/null +++ b/packages/server/postgres/migrations/1709934935000_embeddingsMetadataId.ts @@ -0,0 +1,62 @@ +import {Client} from 'pg' +import getPgConfig from '../getPgConfig' + +export async function up() { + const client = new Client(getPgConfig()) + await client.connect() + // wipe data to ensure the non-null constraints succeed + await client.query(` + CREATE TYPE "ISO6391Enum" AS ENUM ('aa', 'ab', 'af', 'ak', 'am', 'ar', 'an', 'as', 'av', 'ae', 'ay', 'az', 'ba', 'bm', 'be', 'bn', 'bi', 'bo', 'bs', 'br', 'bg', 'ca', 'cs', 'ch', 'ce', 'cu', 'cv', 'kw', 'co', 'cr', 'cy', 'da', 'de', 'dv', 'dz', 'el', 'en', 'eo', 'et', 'eu', 'ee', 'fo', 'fa', 'fj', 'fi', 'fr', 'fy', 'ff', 'gd', 'ga', 'gl', 'gv', 'gn', 'gu', 'ht', 'ha', 'sh', 'he', 'hz', 'hi', 'ho', 'hr', 'hu', 'hy', 'ig', 'io', 'ii', 'iu', 'ie', 'ia', 'id', 'ik', 'is', 'it', 'jv', 'ja', 'kl', 'kn', 'ks', 'ka', 'kr', 'kk', 'km', 'ki', 'rw', 'ky', 'kv', 'kg', 'ko', 'kj', 'ku', 'lo', 'la', 'lv', 'li', 'ln', 'lt', 'lb', 'lu', 'lg', 'mh', 'ml', 'mr', 'mk', 'mg', 'mt', 'mn', 'mi', 'ms', 'my', 'na', 'nv', 'nr', 'nd', 'ng', 'ne', 'nl', 'nn', 'nb', 'no', 'ny', 'oc', 'oj', 'or', 'om', 'os', 'pa', 'pi', 'pl', 'pt', 'ps', 'qu', 'rm', 'ro', 'rn', 'ru', 'sg', 'sa', 'si', 'sk', 'sl', 'se', 'sm', 'sn', 'sd', 'so', 'st', 'es', 'sq', 'sc', 'sr', 'ss', 'su', 'sw', 'sv', 'ty', 'ta', 'tt', 'te', 'tg', 'tl', 'th', 'ti', 'to', 'tn', 'ts', 'tk', 'tr', 'tw', 'ug', 'uk', 'ur', 'uz', 've', 'vi', 'vo', 'wa', 'wo', 'xh', 'yi', 'yo', 'za', 'zh', 'zu'); + DELETE FROM "EmbeddingsMetadata"; + DELETE FROM "EmbeddingsJobQueue"; + CREATE INDEX IF NOT EXISTS "idx_Discussion_createdAt" ON "Discussion"("createdAt"); + ALTER TYPE "EmbeddingsStateEnum" RENAME VALUE 'embedding' TO 'running'; + ALTER TYPE "EmbeddingsStateEnum" RENAME TO "EmbeddingsJobStateEnum"; + ALTER TABLE "EmbeddingsMetadata" + DROP COLUMN "models", + ADD COLUMN "language" "ISO6391Enum", + ALTER COLUMN "refId" SET NOT NULL; + ALTER TABLE "EmbeddingsJobQueue" + ADD COLUMN "retryAfter" TIMESTAMP WITH TIME ZONE, + ADD COLUMN "retryCount" SMALLINT NOT NULL DEFAULT 0, + ADD COLUMN "startAt" TIMESTAMP WITH TIME ZONE, + ADD COLUMN "priority" SMALLINT NOT NULL DEFAULT 50, + ADD COLUMN "jobData" JSONB NOT NULL DEFAULT '{}', + ADD COLUMN "jobType" VARCHAR(255) NOT NULL, + DROP CONSTRAINT IF EXISTS "EmbeddingsJobQueue_objectType_refId_model_key", + DROP COLUMN "refId", + DROP COLUMN "objectType", + DROP COLUMN "model"; + CREATE INDEX IF NOT EXISTS "idx_EmbeddingsJobQueue_priority" ON "EmbeddingsJobQueue"("priority"); + `) + await client.end() +} + +export async function down() { + const client = new Client(getPgConfig()) + await client.connect() + await client.query(` + DROP TYPE IF EXISTS "ISO6391Enum" CASCADE; + DROP INDEX IF EXISTS "idx_Discussion_createdAt"; + ALTER TYPE "EmbeddingsJobStateEnum" RENAME VALUE 'running' TO 'embedding'; + ALTER TYPE "EmbeddingsJobStateEnum" RENAME TO "EmbeddingsStateEnum"; + ALTER TABLE "EmbeddingsMetadata" + ADD COLUMN "models" VARCHAR(255)[], + DROP COLUMN IF EXISTS "language", + ALTER COLUMN "refId" DROP NOT NULL; + ALTER TABLE "EmbeddingsJobQueue" + ALTER COLUMN "state" TYPE "EmbeddingsStateEnum" USING "state"::text::"EmbeddingsStateEnum", + ADD CONSTRAINT "EmbeddingsJobQueue_objectType_refId_model_key" UNIQUE("objectType", "refId", "model"), + ADD COLUMN "refId" VARCHAR(100), + ADD COLUMN "model" VARCHAR(255), + ADD COLUMN "objectType" "EmbeddingsObjectTypeEnum", + DROP COLUMN "retryAfter", + DROP COLUMN "retryCount", + DROP COLUMN "startAt", + DROP COLUMN "jobData", + DROP COLUMN "priority", + DROP COLUMN "jobType"; + DROP INDEX IF EXISTS "idx_EmbeddingsJobQueue_priority"; + `) + await client.end() +} diff --git a/packages/server/postgres/types/Meeting.ts b/packages/server/postgres/types/Meeting.d.ts similarity index 100% rename from packages/server/postgres/types/Meeting.ts rename to packages/server/postgres/types/Meeting.d.ts diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index a1a9c3130df..bc89d53cbbe 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -8,24 +8,16 @@ "parabol-client/*": ["client/*"], "~/*": ["client/*"] }, - "lib": ["esnext", "dom"], - "types": ["node", "jest", "jest-extended"] + "lib": ["esnext", "dom"] }, "exclude": [ "**/node_modules", "types/githubTypes.ts", "postgres/migrationTemplate.ts", - "graphql/intranetSchema/sdl/resolverTypes.ts" - ], - "files": [ - "types/webpackEnv.ts", - "types/modules.d.ts", - "server.ts", - "../client/modules/email/components/SummaryEmail/MeetingSummaryEmail/MeetingSummaryEmail.tsx" - ], - "include": ["graphql/**/*.ts"] - - // if "include" or "files" is added, even if they are empty arrays, then strictNullChecks breaks - // repro: https://www.typescriptlang.org/play?strictFunctionTypes=false&strictPropertyInitialization=false&strictBindCallApply=false&noImplicitThis=false&noImplicitReturns=false&alwaysStrict=false&declaration=false&experimentalDecorators=false&emitDecoratorMetadata=false&target=6&ts=3.5.1#code/C4TwDgpgBA8gRgKygXigbwFBSgWwIYhwQDKwATgIJlkD8AXFAM7kCWAdgOYDaAuhgL4ZQkKFTIpYiLgHJ8hEuTHS+AYwD2bZlDwS2AVwA2B7Y21sQJ0dQx4AdADM1ZAKJ4VACwAUngF4BKFAA+KH8MIA + "database/migrations/**/*", + "postgres/migrations/**/*", + "graphql/intranetSchema/sdl/resolverTypes.ts", + "billing/debug.ts" + ] } diff --git a/packages/server/types/custom.d.ts b/packages/server/types/custom.d.ts index 34587f968c4..aae67c270cd 100644 --- a/packages/server/types/custom.d.ts +++ b/packages/server/types/custom.d.ts @@ -1,3 +1,9 @@ +import {GraphQLSchema} from 'graphql' +import type nestGitHubEndpoint from 'nest-graphql-endpoint/lib/nestGitHubEndpoint' +import '../../client/types/reactHTML4' +import type AuthToken from '../database/types/AuthToken' +import type ScheduledJobMeetingStageTimeLimit from '../database/types/ScheduledJobMetingStageTimeLimit' +import type ScheduledTeamLimitsJob from '../database/types/ScheduledTeamLimitsJob' export interface OAuth2Success { access_token: string token_type: string @@ -17,3 +23,26 @@ export interface OAuth2Error { error_description?: string error_uri?: string } +export interface GQLRequest { + authToken: AuthToken + ip?: string + socketId?: string + variables?: {[key: string]: any} + docId?: string + query?: string + rootValue?: {[key: string]: any} + dataLoaderId?: string + // true if the query is on the private schema + isPrivate?: boolean + // true if the query is ad-hoc (e.g. GraphiQL, CLI) + isAdHoc?: boolean + // Datadog opentracing span of the calling server + carrier?: any +} + +export type ScheduledJobUnion = ScheduledJobMeetingStageTimeLimit | ScheduledTeamLimitsJob + +export type RootSchema = GraphQLSchema & { + githubRequest: ReturnType['githubRequest'] + gitlabRequest: ReturnType['githubRequest'] +} diff --git a/packages/server/types/modules.d.ts b/packages/server/types/modules.d.ts index c79e5a0b011..9b16b604830 100644 --- a/packages/server/types/modules.d.ts +++ b/packages/server/types/modules.d.ts @@ -21,6 +21,7 @@ declare module 'node-env-flag' declare module '*getProjectRoot' declare module 'tayden-clusterfck' declare module 'unicode-substring' +declare module 'jest-extended' declare module 'json2csv/lib/JSON2CSVParser' declare module 'object-hash' declare module 'string-score' diff --git a/packages/server/utils/__tests__/isRequestToJoinDomainAllowed.test.ts b/packages/server/utils/__tests__/isRequestToJoinDomainAllowed.test.ts index 3c65dff93cb..6858d2854cf 100644 --- a/packages/server/utils/__tests__/isRequestToJoinDomainAllowed.test.ts +++ b/packages/server/utils/__tests__/isRequestToJoinDomainAllowed.test.ts @@ -1,16 +1,16 @@ /* eslint-env jest */ -import {MasterPool, r} from 'rethinkdb-ts' -import getRedis from '../getRedis' -import RedisLockQueue from '../RedisLockQueue' -import sleep from 'parabol-client/utils/sleep' +import {r} from 'rethinkdb-ts' import getRethinkConfig from '../../database/getRethinkConfig' import getRethink from '../../database/rethinkDriver' -import {getEligibleOrgIdsByDomain} from '../isRequestToJoinDomainAllowed' +import OrganizationUser from '../../database/types/OrganizationUser' import generateUID from '../../generateUID' +import {DataLoaderWorker} from '../../graphql/graphql' +import getRedis from '../getRedis' +import {getEligibleOrgIdsByDomain} from '../isRequestToJoinDomainAllowed' jest.mock('../../database/rethinkDriver') -getRethink.mockImplementation(() => { - return r +jest.mocked(getRethink).mockImplementation(() => { + return r as any }) const TEST_DB = 'isRequestToJoinDomainAllowedTest' @@ -21,7 +21,7 @@ const testConfig = { db: TEST_DB } -const createTables = async (...tables: string) => { +const createTables = async (...tables: string[]) => { for (const tableName of tables) { const structure = await r .db('rethinkdb') @@ -37,9 +37,8 @@ const createTables = async (...tables: string) => { } } -type TestOrganizationUser = Pick< - OrganizationUser, - 'inactive' | 'joinedAt' | 'removedAt' | 'role' | 'userId' +type TestOrganizationUser = Partial< + Pick > const addOrg = async ( @@ -88,12 +87,12 @@ const dataLoader = { users: userLoader, isCompanyDomain: isCompanyDomainLoader } - return loaders[loader] + return loaders[loader as keyof typeof loaders] }) -} +} as any as DataLoaderWorker beforeAll(async () => { - const conn = await r.connectPool(testConfig) + await r.connectPool(testConfig) try { await r.dbDrop(TEST_DB).run() } catch (e) { @@ -109,7 +108,7 @@ afterEach(async () => { }) afterAll(async () => { - await r.getPoolMaster().drain() + await r.getPoolMaster()?.drain() getRedis().quit() }) @@ -126,7 +125,7 @@ test('Founder is billing lead', async () => { } ]) - const orgIds = await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) expect(userLoader.loadMany).toHaveBeenCalledTimes(1) expect(userLoader.loadMany).toHaveBeenCalledWith(['user1']) }) @@ -148,7 +147,7 @@ test('Org with noPromptToJoinOrg feature flag is ignored', async () => { {featureFlags: ['noPromptToJoinOrg']} ) - const orgIds = await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) expect(userLoader.loadMany).toHaveBeenCalledTimes(0) }) @@ -170,7 +169,7 @@ test('Inactive founder is ignored', async () => { } ]) - const orgIds = await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) // implementation detail, important is only that no user was loaded expect(userLoader.loadMany).toHaveBeenCalledTimes(1) expect(userLoader.loadMany).toHaveBeenCalledWith([]) @@ -195,7 +194,7 @@ test('Non-founder billing lead is checked', async () => { } ]) - const orgIds = await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) expect(userLoader.loadMany).toHaveBeenCalledTimes(1) expect(userLoader.loadMany).toHaveBeenCalledWith(['billing1']) }) @@ -212,7 +211,7 @@ test('Founder is checked even when not billing lead', async () => { } ]) - const orgIds = await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) expect(userLoader.loadMany).toHaveBeenCalledTimes(1) expect(userLoader.loadMany).toHaveBeenCalledWith(['user1']) }) @@ -239,7 +238,7 @@ test('All matching orgs are checked', async () => { } ]) - const orgIds = await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) // implementation detail, important is only that both users were loaded expect(userLoader.loadMany).toHaveBeenCalledTimes(2) expect(userLoader.loadMany).toHaveBeenCalledWith(['founder1']) @@ -249,12 +248,12 @@ test('All matching orgs are checked', async () => { test('Empty org does not throw', async () => { await addOrg('parabol.co', []) - const orgIds = await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) expect(userLoader.loadMany).toHaveBeenCalledTimes(0) }) test('No org does not throw', async () => { - const orgIds = await getEligibleOrgIdsByDomain('example.com', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('example.com', 'newUser', dataLoader) expect(userLoader.loadMany).toHaveBeenCalledTimes(0) }) @@ -267,7 +266,7 @@ test('1 person orgs are ignored', async () => { } ]) - const orgIds = await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) expect(userLoader.loadMany).toHaveBeenCalledTimes(0) }) @@ -283,12 +282,12 @@ test('Org matching the user are ignored', async () => { } ]) - const orgIds = await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) + await getEligibleOrgIdsByDomain('parabol.co', 'newUser', dataLoader) expect(userLoader.loadMany).toHaveBeenCalledTimes(0) }) test('Only the biggest org with verified emails qualify', async () => { - const org = await addOrg('parabol.co', [ + await addOrg('parabol.co', [ { joinedAt: new Date('2023-09-06'), userId: 'founder1' @@ -350,7 +349,7 @@ test('Only the biggest org with verified emails qualify', async () => { ] } } - return userIds.map((id) => ({ + return userIds.map((id: keyof typeof users) => ({ id, ...users[id] })) @@ -423,7 +422,7 @@ test('All the biggest orgs with verified emails qualify', async () => { ] } } - return userIds.map((id) => ({ + return userIds.map((id: keyof typeof users) => ({ id, ...users[id] })) @@ -452,7 +451,7 @@ test('Team trumps starter tier with more users org', async () => { ], {tier: 'team'} ) - const biggerStarterOrg = await addOrg('parabol.co', [ + await addOrg('parabol.co', [ { joinedAt: new Date('2023-09-06'), userId: 'founder2' @@ -504,7 +503,7 @@ test('Team trumps starter tier with more users org', async () => { ] } } - return userIds.map((id) => ({ + return userIds.map((id: keyof typeof users) => ({ id, ...users[id] })) @@ -533,7 +532,7 @@ test('Enterprise trumps team tier with more users org', async () => { ], {tier: 'enterprise'} ) - const starterOrg = await addOrg( + await addOrg( 'parabol.co', [ { @@ -589,7 +588,7 @@ test('Enterprise trumps team tier with more users org', async () => { ] } } - return userIds.map((id) => ({ + return userIds.map((id: keyof typeof users) => ({ id, ...users[id] })) @@ -604,7 +603,7 @@ test('Enterprise trumps team tier with more users org', async () => { }) test('Orgs with verified emails from different domains do not qualify', async () => { - const org1 = await addOrg('parabol.co', [ + await addOrg('parabol.co', [ { joinedAt: new Date('2023-09-06'), userId: 'founder1' diff --git a/packages/server/utils/__tests__/rateLimiters/InMemoryRateLimiter.test.ts b/packages/server/utils/__tests__/rateLimiters/InMemoryRateLimiter.test.ts index 01f4fa4394a..3b2478aa9d6 100644 --- a/packages/server/utils/__tests__/rateLimiters/InMemoryRateLimiter.test.ts +++ b/packages/server/utils/__tests__/rateLimiters/InMemoryRateLimiter.test.ts @@ -14,7 +14,7 @@ describe('InMemoryRateLimiter', () => { } beforeEach(() => { - jest.useFakeTimers('modern') + jest.useFakeTimers() }) afterEach(() => { diff --git a/packages/server/utils/analytics/analytics.ts b/packages/server/utils/analytics/analytics.ts index a4cbc61f189..305839e2651 100644 --- a/packages/server/utils/analytics/analytics.ts +++ b/packages/server/utils/analytics/analytics.ts @@ -5,9 +5,12 @@ import Meeting from '../../database/types/Meeting' import MeetingMember from '../../database/types/MeetingMember' import MeetingRetrospective from '../../database/types/MeetingRetrospective' import MeetingTemplate from '../../database/types/MeetingTemplate' -import {Reactable} from '../../database/types/Reactable' +import {Reactable, ReactableEnum} from '../../database/types/Reactable' +import {SlackNotificationEventEnum} from '../../database/types/SlackNotification' import {TaskServiceEnum} from '../../database/types/Task' -import {ReactableEnum} from '../../graphql/private/resolverTypes' +import TemplateScale from '../../database/types/TemplateScale' +import {DataLoaderWorker} from '../../graphql/graphql' +import {ModifyType} from '../../graphql/public/resolverTypes' import {IntegrationProviderServiceEnumType} from '../../graphql/types/IntegrationProviderServiceEnum' import {UpgradeCTALocationEnumType} from '../../graphql/types/UpgradeCTALocationEnum' import {TeamPromptResponse} from '../../postgres/queries/getTeamPromptResponsesByIds' @@ -16,10 +19,6 @@ import {MeetingTypeEnum} from '../../postgres/types/Meeting' import {MeetingSeries} from '../../postgres/types/MeetingSeries' import {AmplitudeAnalytics} from './amplitude/AmplitudeAnalytics' import {createMeetingProperties} from './helpers' -import {SlackNotificationEventEnum} from '../../database/types/SlackNotification' -import TemplateScale from '../../database/types/TemplateScale' -import {DataLoaderWorker} from '../../graphql/graphql' -import {ModifyType} from '../../graphql/public/resolverTypes' export type AnalyticsUser = { id: string diff --git a/packages/server/utils/getGitHubRequest.ts b/packages/server/utils/getGitHubRequest.ts index 76efb0db5ae..59ff1eea559 100644 --- a/packages/server/utils/getGitHubRequest.ts +++ b/packages/server/utils/getGitHubRequest.ts @@ -1,6 +1,6 @@ import {GraphQLResolveInfo} from 'graphql' import {EndpointContext} from 'nest-graphql-endpoint/lib/types' -import {RootSchema} from '../graphql/public/rootSchema' +import {RootSchema} from '../types/custom' // This helper just cleans up the input/output boilerplate. // It breaks githubRequest into 2 parts so the info, endpointContext, and batchRef are kept in context diff --git a/packages/server/utils/getGraphQLExecutor.ts b/packages/server/utils/getGraphQLExecutor.ts index bc70b875316..759f21bade2 100644 --- a/packages/server/utils/getGraphQLExecutor.ts +++ b/packages/server/utils/getGraphQLExecutor.ts @@ -1,7 +1,7 @@ import {ExecutionResult} from 'graphql' import {ServerChannel} from 'parabol-client/types/constEnums' +import type {GQLRequest} from '../types/custom' import SocketServerChannelId from '../../client/shared/gqlIds/SocketServerChannelId' -import {GQLRequest} from '../graphql/executeGraphQL' import PubSubPromise from './PubSubPromise' let pubsub: PubSubPromise diff --git a/packages/server/utils/uwsGetHeaders.ts b/packages/server/utils/uwsGetHeaders.ts index 9fd7bdcdc02..f51fd27e78f 100644 --- a/packages/server/utils/uwsGetHeaders.ts +++ b/packages/server/utils/uwsGetHeaders.ts @@ -1,7 +1,7 @@ import {HttpRequest} from 'uWebSockets.js' const uwsGetHeaders = (req: HttpRequest) => { - const reqHeaders = {} + const reqHeaders: Record = {} req.forEach((key, value) => { reqHeaders[key] = value }) diff --git a/release-please-config.json b/release-please-config.json index 29f302183a1..b8959848b5a 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -60,6 +60,11 @@ "path": "packages/integration-tests/package.json", "jsonpath": "$.version" }, + { + "type": "json", + "path": "packages/embedder/package.json", + "jsonpath": "$.version" + }, { "type": "json", "path": "packages/server/package.json", diff --git a/scripts/webpack/utils/getProjectRoot.d.ts b/scripts/webpack/utils/getProjectRoot.d.ts new file mode 100644 index 00000000000..436f1feb4c9 --- /dev/null +++ b/scripts/webpack/utils/getProjectRoot.d.ts @@ -0,0 +1,2 @@ +declare function getProjectRoot(): string +export default getProjectRoot diff --git a/tsconfig.base.json b/tsconfig.base.json index adc1623c1b5..c207d6f1c07 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -16,6 +16,7 @@ "strictPropertyInitialization": true, "noImplicitThis": true, "alwaysStrict": true, + "noEmit": true, "noUnusedLocals": true, "noUnusedParameters": true, "noImplicitReturns": true, diff --git a/yarn.lock b/yarn.lock index dc34fd16fae..6701230bae9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@aashutoshrathi/word-wrap@^1.2.3": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf" + integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA== + "@amplitude/analytics-browser@^2.2.3": version "2.2.3" resolved "https://registry.yarnpkg.com/@amplitude/analytics-browser/-/analytics-browser-2.2.3.tgz#7dbe4ad2ada7facfcf21aac4b47314f8e4c2903c" @@ -3166,33 +3171,38 @@ resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.3.0.tgz#ea89004119dc42db2e1dba0f97d553f7372f6fcb" integrity sha512-AHPmaAx+RYfZz0eYu6Gviiagpmiyw98ySSlQvCUhVGDRtDFe4DBS0x1bSjdF3gqUDYOczB+yYvBTtEylYSdRhg== -"@eslint-community/eslint-utils@^4.4.0": +"@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": version "4.4.0" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" integrity sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA== dependencies: eslint-visitor-keys "^3.3.0" -"@eslint-community/regexpp@^4.5.1": +"@eslint-community/regexpp@^4.5.1", "@eslint-community/regexpp@^4.6.1": version "4.10.0" resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.10.0.tgz#548f6de556857c8bb73bbee70c35dc82a2e74d63" integrity sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA== -"@eslint/eslintrc@^1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.0.5.tgz#33f1b838dbf1f923bfa517e008362b78ddbbf318" - integrity sha512-BLxsnmK3KyPunz5wmCCpqy0YelEoxxGmH73Is+Z74oOTMtExcjkr3dDR6quwrjh1YspA8DH9gnX1o069KiS9AQ== +"@eslint/eslintrc@^2.1.4": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-2.1.4.tgz#388a269f0f25c1b6adc317b5a2c55714894c70ad" + integrity sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ== dependencies: ajv "^6.12.4" debug "^4.3.2" - espree "^9.2.0" - globals "^13.9.0" - ignore "^4.0.6" + espree "^9.6.0" + globals "^13.19.0" + ignore "^5.2.0" import-fresh "^3.2.1" js-yaml "^4.1.0" - minimatch "^3.0.4" + minimatch "^3.1.2" strip-json-comments "^3.1.1" +"@eslint/js@8.57.0": + version "8.57.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.0.tgz#a5417ae8427873f1dd08b70b3574b453e67b5f7f" + integrity sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g== + "@exodus/schemasafe@^1.0.0-rc.2": version "1.0.1" resolved "https://registry.yarnpkg.com/@exodus/schemasafe/-/schemasafe-1.0.1.tgz#e4e2d86ae176b7c96fbff033f3b1a8b1cfd390fb" @@ -3833,24 +3843,29 @@ dependencies: client-only "^0.0.1" -"@humanwhocodes/config-array@^0.9.2": - version "0.9.2" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.9.2.tgz#68be55c737023009dfc5fe245d51181bb6476914" - integrity sha512-UXOuFCGcwciWckOpmfKDq/GyhlTf9pN/BzG//x8p8zTOFEcGuA68ANXheFS0AGvy3qgZqLBUkMs7hqzqCKOVwA== +"@humanwhocodes/config-array@^0.11.14": + version "0.11.14" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" + integrity sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg== dependencies: - "@humanwhocodes/object-schema" "^1.2.1" - debug "^4.1.1" - minimatch "^3.0.4" + "@humanwhocodes/object-schema" "^2.0.2" + debug "^4.3.1" + minimatch "^3.0.5" + +"@humanwhocodes/module-importer@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" + integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== "@humanwhocodes/momoa@^2.0.3": version "2.0.4" resolved "https://registry.yarnpkg.com/@humanwhocodes/momoa/-/momoa-2.0.4.tgz#8b9e7a629651d15009c3587d07a222deeb829385" integrity sha512-RE815I4arJFtt+FVeU1Tgp9/Xvecacji8w/V6XtXsWWH/wz/eNkNbhb+ny/+PlVZjV0rxQpRSQKNKE3lcktHEA== -"@humanwhocodes/object-schema@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" - integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@humanwhocodes/object-schema@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz#d9fae00a2d5cb40f92cfe64b47ad749fbc38f917" + integrity sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw== "@hutson/parse-repository-url@^3.0.0": version "3.0.2" @@ -5065,7 +5080,7 @@ resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b" integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== -"@nodelib/fs.walk@^1.2.3": +"@nodelib/fs.walk@^1.2.3", "@nodelib/fs.walk@^1.2.8": version "1.2.8" resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a" integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== @@ -7565,6 +7580,11 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/faker@^5.5.9": + version "5.5.9" + resolved "https://registry.yarnpkg.com/@types/faker/-/faker-5.5.9.tgz#588ede92186dc557bff8341d294335d50d255f0c" + integrity sha512-uCx6mP3UY5SIO14XlspxsGjgaemrxpssJI0Ol+GfhxtcKpv9pgRZYsS4eeKeHVLje6Qtc8lGszuBI461+gVZBA== + "@types/fbjs@^3.0.4": version "3.0.4" resolved "https://registry.yarnpkg.com/@types/fbjs/-/fbjs-3.0.4.tgz#272faf44b6a24d24d6fdf36ed895b47436e6c125" @@ -7572,6 +7592,11 @@ dependencies: "@types/jsdom" "*" +"@types/franc@^5.0.3": + version "5.0.3" + resolved "https://registry.yarnpkg.com/@types/franc/-/franc-5.0.3.tgz#7263cef3ab3512ac95a78c328fcc51c51396b49f" + integrity sha512-YX6o2vVkeiUvOF12bUmnSGf8sezOoBnCWjHHZGeh2lt3tqAutbJ9OL3cDRiZoiAYaZR638nuOc0Ji9bzdad2XA== + "@types/glob@*": version "8.1.0" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-8.1.0.tgz#b63e70155391b0584dce44e7ea25190bbc38f2fc" @@ -7674,6 +7699,14 @@ expect "^29.0.0" pretty-format "^29.0.0" +"@types/jest@^29.5.12": + version "29.5.12" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.5.12.tgz#7f7dc6eb4cf246d2474ed78744b05d06ce025544" + integrity sha512-eDC8bTvT/QhYdxJAulQikueigY5AsdBRH2yDKW3yveW7svY3+DzN84/2NUgkw10RTiJbWqZrTtoGVdYlvFJdLw== + dependencies: + expect "^29.0.0" + pretty-format "^29.0.0" + "@types/js-yaml@^4.0.0": version "4.0.5" resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.5.tgz#738dd390a6ecc5442f35e7f03fa1431353f7e138" @@ -8190,16 +8223,16 @@ resolved "https://registry.yarnpkg.com/@types/yup/-/yup-0.29.11.tgz#d654a112973f5e004bf8438122bd7e56a8e5cd7e" integrity sha512-9cwk3c87qQKZrT251EDoibiYRILjCmxBvvcb4meofCmx1vdnNcR9gyildy5vOHASpOKMsn42CugxUvcwK5eu1g== -"@typescript-eslint/eslint-plugin@^6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.21.0.tgz#30830c1ca81fd5f3c2714e524c4303e0194f9cd3" - integrity sha512-oy9+hTPCUFpngkEZUSzbf9MxI65wbKFoQYsgPdILTfbUldp5ovUuphZVe4i30emU9M/kP+T64Di0mxl7dSw3MA== +"@typescript-eslint/eslint-plugin@^7.4.0": + version "7.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.4.0.tgz#de61c3083842fc6ac889d2fc83c9a96b55ab8328" + integrity sha512-yHMQ/oFaM7HZdVrVm/M2WHaNPgyuJH4WelkSVEWSSsir34kxW2kDJCxlXRhhGWEsMN0WAW/vLpKfKVcm8k+MPw== dependencies: "@eslint-community/regexpp" "^4.5.1" - "@typescript-eslint/scope-manager" "6.21.0" - "@typescript-eslint/type-utils" "6.21.0" - "@typescript-eslint/utils" "6.21.0" - "@typescript-eslint/visitor-keys" "6.21.0" + "@typescript-eslint/scope-manager" "7.4.0" + "@typescript-eslint/type-utils" "7.4.0" + "@typescript-eslint/utils" "7.4.0" + "@typescript-eslint/visitor-keys" "7.4.0" debug "^4.3.4" graphemer "^1.4.0" ignore "^5.2.4" @@ -8207,47 +8240,47 @@ semver "^7.5.4" ts-api-utils "^1.0.1" -"@typescript-eslint/parser@^6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.21.0.tgz#af8fcf66feee2edc86bc5d1cf45e33b0630bf35b" - integrity sha512-tbsV1jPne5CkFQCgPBcDOt30ItF7aJoZL997JSF7MhGQqOeT3svWRYxiqlfA5RUdlHN6Fi+EI9bxqbdyAUZjYQ== +"@typescript-eslint/parser@^7.4.0": + version "7.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-7.4.0.tgz#540f4321de1e52b886c0fa68628af1459954c1f1" + integrity sha512-ZvKHxHLusweEUVwrGRXXUVzFgnWhigo4JurEj0dGF1tbcGh6buL+ejDdjxOQxv6ytcY1uhun1p2sm8iWStlgLQ== dependencies: - "@typescript-eslint/scope-manager" "6.21.0" - "@typescript-eslint/types" "6.21.0" - "@typescript-eslint/typescript-estree" "6.21.0" - "@typescript-eslint/visitor-keys" "6.21.0" + "@typescript-eslint/scope-manager" "7.4.0" + "@typescript-eslint/types" "7.4.0" + "@typescript-eslint/typescript-estree" "7.4.0" + "@typescript-eslint/visitor-keys" "7.4.0" debug "^4.3.4" -"@typescript-eslint/scope-manager@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.21.0.tgz#ea8a9bfc8f1504a6ac5d59a6df308d3a0630a2b1" - integrity sha512-OwLUIWZJry80O99zvqXVEioyniJMa+d2GrqpUTqi5/v5D5rOrppJVBPa0yKCblcigC0/aYAzxxqQ1B+DS2RYsg== +"@typescript-eslint/scope-manager@7.4.0": + version "7.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-7.4.0.tgz#acfc69261f10ece7bf7ece1734f1713392c3655f" + integrity sha512-68VqENG5HK27ypafqLVs8qO+RkNc7TezCduYrx8YJpXq2QGZ30vmNZGJJJC48+MVn4G2dCV8m5ZTVnzRexTVtw== dependencies: - "@typescript-eslint/types" "6.21.0" - "@typescript-eslint/visitor-keys" "6.21.0" + "@typescript-eslint/types" "7.4.0" + "@typescript-eslint/visitor-keys" "7.4.0" -"@typescript-eslint/type-utils@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.21.0.tgz#6473281cfed4dacabe8004e8521cee0bd9d4c01e" - integrity sha512-rZQI7wHfao8qMX3Rd3xqeYSMCL3SoiSQLBATSiVKARdFGCYSRvmViieZjqc58jKgs8Y8i9YvVVhRbHSTA4VBag== +"@typescript-eslint/type-utils@7.4.0": + version "7.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-7.4.0.tgz#cfcaab21bcca441c57da5d3a1153555e39028cbd" + integrity sha512-247ETeHgr9WTRMqHbbQdzwzhuyaJ8dPTuyuUEMANqzMRB1rj/9qFIuIXK7l0FX9i9FXbHeBQl/4uz6mYuCE7Aw== dependencies: - "@typescript-eslint/typescript-estree" "6.21.0" - "@typescript-eslint/utils" "6.21.0" + "@typescript-eslint/typescript-estree" "7.4.0" + "@typescript-eslint/utils" "7.4.0" debug "^4.3.4" ts-api-utils "^1.0.1" -"@typescript-eslint/types@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.21.0.tgz#205724c5123a8fef7ecd195075fa6e85bac3436d" - integrity sha512-1kFmZ1rOm5epu9NZEZm1kckCDGj5UJEf7P1kliH4LKu/RkwpsfqqGmY2OOcUs18lSlQBKLDYBOGxRVtrMN5lpg== +"@typescript-eslint/types@7.4.0": + version "7.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.4.0.tgz#ee9dafa75c99eaee49de6dcc9348b45d354419b6" + integrity sha512-mjQopsbffzJskos5B4HmbsadSJQWaRK0UxqQ7GuNA9Ga4bEKeiO6b2DnB6cM6bpc8lemaPseh0H9B/wyg+J7rw== -"@typescript-eslint/typescript-estree@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.21.0.tgz#c47ae7901db3b8bddc3ecd73daff2d0895688c46" - integrity sha512-6npJTkZcO+y2/kr+z0hc4HwNfrrP4kNYh57ek7yCNlrBjWQ1Y0OS7jiZTkgumrvkX5HkEKXFZkkdFNkaW2wmUQ== +"@typescript-eslint/typescript-estree@7.4.0": + version "7.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-7.4.0.tgz#12dbcb4624d952f72c10a9f4431284fca24624f4" + integrity sha512-A99j5AYoME/UBQ1ucEbbMEmGkN7SE0BvZFreSnTd1luq7yulcHdyGamZKizU7canpGDWGJ+Q6ZA9SyQobipePg== dependencies: - "@typescript-eslint/types" "6.21.0" - "@typescript-eslint/visitor-keys" "6.21.0" + "@typescript-eslint/types" "7.4.0" + "@typescript-eslint/visitor-keys" "7.4.0" debug "^4.3.4" globby "^11.1.0" is-glob "^4.0.3" @@ -8255,27 +8288,32 @@ semver "^7.5.4" ts-api-utils "^1.0.1" -"@typescript-eslint/utils@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.21.0.tgz#4714e7a6b39e773c1c8e97ec587f520840cd8134" - integrity sha512-NfWVaC8HP9T8cbKQxHcsJBY5YE1O33+jpMwN45qzWWaPDZgLIbo12toGMWnmhvCpd3sIxkpDw3Wv1B3dYrbDQQ== +"@typescript-eslint/utils@7.4.0": + version "7.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-7.4.0.tgz#d889a0630cab88bddedaf7c845c64a00576257bd" + integrity sha512-NQt9QLM4Tt8qrlBVY9lkMYzfYtNz8/6qwZg8pI3cMGlPnj6mOpRxxAm7BMJN9K0AiY+1BwJ5lVC650YJqYOuNg== dependencies: "@eslint-community/eslint-utils" "^4.4.0" "@types/json-schema" "^7.0.12" "@types/semver" "^7.5.0" - "@typescript-eslint/scope-manager" "6.21.0" - "@typescript-eslint/types" "6.21.0" - "@typescript-eslint/typescript-estree" "6.21.0" + "@typescript-eslint/scope-manager" "7.4.0" + "@typescript-eslint/types" "7.4.0" + "@typescript-eslint/typescript-estree" "7.4.0" semver "^7.5.4" -"@typescript-eslint/visitor-keys@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.21.0.tgz#87a99d077aa507e20e238b11d56cc26ade45fe47" - integrity sha512-JJtkDduxLi9bivAB+cYOVMtbkqdPOhZ+ZI5LC47MIRrDV4Yn2o+ZnW10Nkmr28xRpSpdJ6Sm42Hjf2+REYXm0A== +"@typescript-eslint/visitor-keys@7.4.0": + version "7.4.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-7.4.0.tgz#0c8ff2c1f8a6fe8d7d1a57ebbd4a638e86a60a94" + integrity sha512-0zkC7YM0iX5Y41homUUeW1CHtZR01K3ybjM1l6QczoMuay0XKtrb93kv95AxUGwdjGr64nNqnOCwmEl616N8CA== dependencies: - "@typescript-eslint/types" "6.21.0" + "@typescript-eslint/types" "7.4.0" eslint-visitor-keys "^3.4.1" +"@ungap/structured-clone@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" + integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== + "@webassemblyjs/ast@1.11.5", "@webassemblyjs/ast@^1.11.5": version "1.11.5" resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.5.tgz#6e818036b94548c1fb53b754b5cae3c9b208281c" @@ -8672,7 +8710,7 @@ ajv-keywords@^5.0.0: dependencies: fast-deep-equal "^3.1.3" -ajv@^6.10.0, ajv@^6.12.4, ajv@^6.12.5: +ajv@^6.12.4, ajv@^6.12.5: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -9704,9 +9742,9 @@ camelcase@^6.2.0: integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== caniuse-lite@^1.0.30001426, caniuse-lite@^1.0.30001517, caniuse-lite@^1.0.30001580, caniuse-lite@~1.0.0: - version "1.0.30001594" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001594.tgz#bea552414cd52c2d0c985ed9206314a696e685f5" - integrity sha512-VblSX6nYqyJVs8DKFMldE2IVCJjZ225LW00ydtUWwh5hk9IfkTOffO6r8gJNsH0qqqeAF8KrbMYA2VEwTlGW5g== + version "1.0.30001603" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001603.tgz#605046a5bdc95ba4a92496d67e062522dce43381" + integrity sha512-iL2iSS0eDILMb9n5yKQoTBim9jMZ0Yrk8g0N9K7UzYyWnfIKzXBZD5ngpM37ZcL/cv0Mli8XtVMRYMQAfFpi5Q== capital-case@^1.0.4: version "1.0.4" @@ -10096,6 +10134,11 @@ codemirror@^5.65.3: resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-5.65.3.tgz#2d029930d5a293bc5fb96ceea64654803c0d4ac7" integrity sha512-kCC0iwGZOVZXHEKW3NDTObvM7pTIyowjty4BUqeREROc/3I6bWbgZDA3fGDwlA+rbgRjvnRnfqs9SfXynel1AQ== +collapse-white-space@^1.0.0: + version "1.0.6" + resolved "https://registry.yarnpkg.com/collapse-white-space/-/collapse-white-space-1.0.6.tgz#e63629c0016665792060dbbeb79c42239d2c5287" + integrity sha512-jEovNnrhMuqyCcjfEJA56v0Xq8SkIoPKDyaHahwo3POf4qcSXqMYuwNcOTzp74vTsR9Tn08z4MxWqAhcekogkQ== + collect-v8-coverage@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/collect-v8-coverage/-/collect-v8-coverage-1.0.1.tgz#cc2c8e94fc18bbdffe64d6534570c8a673b27f59" @@ -11305,7 +11348,6 @@ draft-js-utils@^1.4.0: "draft-js@https://github.com/mattkrick/draft-js/tarball/559a21968370c4944511657817d601a6c4ade0f6": version "0.10.5" - uid "025fddba56f21aaf3383aee778e0b17025c9a7bc" resolved "https://github.com/mattkrick/draft-js/tarball/559a21968370c4944511657817d601a6c4ade0f6#025fddba56f21aaf3383aee778e0b17025c9a7bc" dependencies: fbjs "^0.8.15" @@ -11639,10 +11681,10 @@ escodegen@^2.0.0, escodegen@^2.1.0: optionalDependencies: source-map "~0.6.1" -eslint-config-prettier@^8.5.0: - version "8.5.0" - resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-8.5.0.tgz#5a81680ec934beca02c7b1a61cf8ca34b66feab1" - integrity sha512-obmWKLUNCnhtQRKc+tmnYuQl0pFU1ibYJQ5BGhTVB08bHe9wC8qUeG7c08dj9XX+AuPj1YSGSQIHl1pnDHZR0Q== +eslint-config-prettier@^9.1.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz#31af3d94578645966c082fcb71a5846d3c94867f" + integrity sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw== eslint-plugin-emotion@^10.0.14: version "10.0.27" @@ -11682,71 +11724,62 @@ eslint-scope@5.1.1: esrecurse "^4.3.0" estraverse "^4.1.1" -eslint-scope@^7.1.0: - version "7.1.0" - resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.1.0.tgz#c1f6ea30ac583031f203d65c73e723b01298f153" - integrity sha512-aWwkhnS0qAXqNOgKOK0dJ2nvzEbhEvpy8OlJ9kZ0FeZnA6zpjv1/Vei+puGFFX7zkPCkHHXb7IDX3A+7yPrRWg== +eslint-scope@^7.2.2: + version "7.2.2" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-7.2.2.tgz#deb4f92563390f32006894af62a22dba1c46423f" + integrity sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg== dependencies: esrecurse "^4.3.0" estraverse "^5.2.0" -eslint-utils@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-3.0.0.tgz#8aebaface7345bb33559db0a1f13a1d2d48c3672" - integrity sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA== - dependencies: - eslint-visitor-keys "^2.0.0" - -eslint-visitor-keys@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz#f65328259305927392c938ed44eb0a5c9b2bd303" - integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw== - -eslint-visitor-keys@^3.2.0, eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1: +eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4.3: version "3.4.3" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== -eslint@^8.2.0, eslint@^8.8.0: - version "8.8.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.8.0.tgz#9762b49abad0cb4952539ffdb0a046392e571a2d" - integrity sha512-H3KXAzQGBH1plhYS3okDix2ZthuYJlQQEGE5k0IKuEqUSiyu4AmxxlJ2MtTYeJ3xB4jDhcYCwGOg2TXYdnDXlQ== - dependencies: - "@eslint/eslintrc" "^1.0.5" - "@humanwhocodes/config-array" "^0.9.2" - ajv "^6.10.0" +eslint@^8.57.0: + version "8.57.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.57.0.tgz#c786a6fd0e0b68941aaf624596fb987089195668" + integrity sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ== + dependencies: + "@eslint-community/eslint-utils" "^4.2.0" + "@eslint-community/regexpp" "^4.6.1" + "@eslint/eslintrc" "^2.1.4" + "@eslint/js" "8.57.0" + "@humanwhocodes/config-array" "^0.11.14" + "@humanwhocodes/module-importer" "^1.0.1" + "@nodelib/fs.walk" "^1.2.8" + "@ungap/structured-clone" "^1.2.0" + ajv "^6.12.4" chalk "^4.0.0" cross-spawn "^7.0.2" debug "^4.3.2" doctrine "^3.0.0" escape-string-regexp "^4.0.0" - eslint-scope "^7.1.0" - eslint-utils "^3.0.0" - eslint-visitor-keys "^3.2.0" - espree "^9.3.0" - esquery "^1.4.0" + eslint-scope "^7.2.2" + eslint-visitor-keys "^3.4.3" + espree "^9.6.1" + esquery "^1.4.2" esutils "^2.0.2" fast-deep-equal "^3.1.3" file-entry-cache "^6.0.1" - functional-red-black-tree "^1.0.1" - glob-parent "^6.0.1" - globals "^13.6.0" + find-up "^5.0.0" + glob-parent "^6.0.2" + globals "^13.19.0" + graphemer "^1.4.0" ignore "^5.2.0" - import-fresh "^3.0.0" imurmurhash "^0.1.4" is-glob "^4.0.0" + is-path-inside "^3.0.3" js-yaml "^4.1.0" json-stable-stringify-without-jsonify "^1.0.1" levn "^0.4.1" lodash.merge "^4.6.2" - minimatch "^3.0.4" + minimatch "^3.1.2" natural-compare "^1.4.0" - optionator "^0.9.1" - regexpp "^3.2.0" + optionator "^0.9.3" strip-ansi "^6.0.1" - strip-json-comments "^3.1.0" text-table "^0.2.0" - v8-compile-cache "^2.0.3" esniff@^2.0.1: version "2.0.1" @@ -11758,7 +11791,7 @@ esniff@^2.0.1: event-emitter "^0.3.5" type "^2.7.2" -espree@^9.0.0, espree@^9.2.0, espree@^9.3.0: +espree@^9.0.0, espree@^9.6.0, espree@^9.6.1: version "9.6.1" resolved "https://registry.yarnpkg.com/espree/-/espree-9.6.1.tgz#a2a17b8e434690a5432f2f8018ce71d331a48c6f" integrity sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ== @@ -11772,13 +11805,20 @@ esprima@^4.0.0, esprima@^4.0.1, esprima@~4.0.0: resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== -esquery@^1.0.1, esquery@^1.4.0: +esquery@^1.0.1: version "1.4.0" resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.4.0.tgz#2148ffc38b82e8c7057dfed48425b3e61f0f24a5" integrity sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w== dependencies: estraverse "^5.1.0" +esquery@^1.4.2: + version "1.5.0" + resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.5.0.tgz#6ce17738de8577694edd7361c57182ac8cb0db0b" + integrity sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg== + dependencies: + estraverse "^5.1.0" + esrecurse@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" @@ -12389,6 +12429,13 @@ framesync@6.0.1: dependencies: tslib "^2.1.0" +franc-min@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/franc-min/-/franc-min-5.0.0.tgz#5625d0570a18564dcbbfa8330254d23549294d9a" + integrity sha512-xy7Iq7uNflbvNU+bkyYWtP+BOHWZle7kT9GM84gEV14b7/7sgq7M7Flf6v1XRflHAuHoshBMveWA6Q+kEXYeHQ== + dependencies: + trigram-utils "^1.0.0" + fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" @@ -12488,11 +12535,6 @@ function-bind@^1.1.2: resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== -functional-red-black-tree@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" - integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc= - fuzzy@^0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/fuzzy/-/fuzzy-0.1.3.tgz#4c76ec2ff0ac1a36a9dccf9a00df8623078d4ed8" @@ -12823,10 +12865,10 @@ globals@^11.1.0: resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== -globals@^13.6.0, globals@^13.9.0: - version "13.12.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-13.12.0.tgz#4d733760304230a0082ed96e21e5c565f898089e" - integrity sha512-uS8X6lSKN2JumVoXrbUz+uG4BYG+eiawqm3qFcT7ammfbUHeCBoJMlHcec/S3krSk73/AE/f0szYFmgAA3kYZg== +globals@^13.19.0: + version "13.24.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-13.24.0.tgz#8432a19d78ce0c1e833949c36adb345400bb1171" + integrity sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ== dependencies: type-fest "^0.20.2" @@ -13491,11 +13533,6 @@ ignore-walk@^5.0.1: dependencies: minimatch "^5.0.1" -ignore@^4.0.6: - version "4.0.6" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc" - integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg== - ignore@^5.0.4, ignore@^5.1.4, ignore@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a" @@ -13531,7 +13568,7 @@ import-fresh@^2.0.0: caller-path "^2.0.0" resolve-from "^3.0.0" -import-fresh@^3.0.0, import-fresh@^3.1.0, import-fresh@^3.2.1: +import-fresh@^3.1.0, import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== @@ -13925,6 +13962,11 @@ is-path-inside@^2.1.0: dependencies: path-is-inside "^1.0.2" +is-path-inside@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" + integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== + is-plain-obj@^1.0.0, is-plain-obj@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e" @@ -15014,10 +15056,10 @@ kysely-codegen@^0.11.0: micromatch "^4.0.5" minimist "^1.2.8" -kysely@^0.27.2: - version "0.27.2" - resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.27.2.tgz#b289ce5e561064ec613a17149b7155783d2b36de" - integrity sha512-DmRvEfiR/NLpgsTbSxma2ldekhsdcd65+MNiKXyd/qj7w7X5e3cLkXxcj+MypsRDjPhHQ/CD5u3Eq1sBYzX0bw== +kysely@^0.27.3: + version "0.27.3" + resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.27.3.tgz#6cc6c757040500b43c4ac596cdbb12be400ee276" + integrity sha512-lG03Ru+XyOJFsjH3OMY6R/9U38IjDPfnOfDgO3ynhbDr+Dz8fak+X6L62vqu3iybQnj+lG84OttBuU9KY3L9kA== launch-editor@^2.6.0: version "2.6.0" @@ -15883,7 +15925,7 @@ minimatch@9.0.3: dependencies: brace-expansion "^2.0.1" -minimatch@^3.0.2, minimatch@^3.0.4, minimatch@^3.0.5: +minimatch@^3.0.2, minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== @@ -16127,6 +16169,11 @@ mz@^2.7.0: object-assign "^4.0.1" thenify-all "^1.0.0" +n-gram@^1.0.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/n-gram/-/n-gram-1.1.2.tgz#69c609a5c83bb32f82774c9e297f8494c7326798" + integrity sha512-mBTpWKp0NHdujHmxrskPg2jc108mjyMmVxHN1rZGK/ogTLi9O0debDIXlQPqotNELdNmVGtL4jr7SCig+4OWvQ== + nan@^2.17.0: version "2.18.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.18.0.tgz#26a6faae7ffbeb293a39660e88a76b82e30b7554" @@ -16864,11 +16911,23 @@ openai@^4.24.1: node-fetch "^2.6.7" web-streams-polyfill "^3.2.1" +openapi-fetch@^0.9.3: + version "0.9.3" + resolved "https://registry.yarnpkg.com/openapi-fetch/-/openapi-fetch-0.9.3.tgz#37c1dbde7faec885eaa40f351cab1c231b794761" + integrity sha512-tC1NDn71vJHeCzu+lYdrnIpgRt4GxR0B4eSwXNb15ypWpZcpaEOwHFkoz8FcfG5Fvqkz2P0Fl9zQF1JJwBjuvA== + dependencies: + openapi-typescript-helpers "^0.0.7" + openapi-types@^12.1.0: version "12.1.1" resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-12.1.1.tgz#0aface4e05ba60efbf51153ed6af23988796617d" integrity sha512-m/DJaEqOUDSU8KoI74E6A3TokccuDOJ81ewZ6kLFwUT1KEIE0GDWvErtnJJDU4sySx8JKF5kk2GzHUuK6f+VHA== +openapi-typescript-helpers@^0.0.7: + version "0.0.7" + resolved "https://registry.yarnpkg.com/openapi-typescript-helpers/-/openapi-typescript-helpers-0.0.7.tgz#1d0ead67c35864d189c2cb2d0556854ccbb16c38" + integrity sha512-7nwlAtdA1fULipibFRBWE/rnF114q6ejRYzNvhdA/x+qTWAZhXGLc/368dlwMlyJDvCQMCnADjpzb5BS5ZmNSA== + opener@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598" @@ -16891,17 +16950,17 @@ optionator@^0.8.1: type-check "~0.3.2" word-wrap "~1.2.3" -optionator@^0.9.1: - version "0.9.1" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" - integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== +optionator@^0.9.3: + version "0.9.3" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.3.tgz#007397d44ed1872fdc6ed31360190f81814e2c64" + integrity sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg== dependencies: + "@aashutoshrathi/word-wrap" "^1.2.3" deep-is "^0.1.3" fast-levenshtein "^2.0.6" levn "^0.4.1" prelude-ls "^1.2.1" type-check "^0.4.0" - word-wrap "^1.2.3" ora@5.4.1, ora@^5.4.1: version "5.4.1" @@ -18804,11 +18863,6 @@ regexp.prototype.flags@^1.3.1: call-bind "^1.0.2" define-properties "^1.1.3" -regexpp@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" - integrity sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg== - regexpu-core@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.1.0.tgz#2f8504c3fd0ebe11215783a41541e21c79942c6d" @@ -20384,7 +20438,19 @@ tar@^4.4.13: safe-buffer "^5.2.1" yallist "^3.1.1" -tar@^6.0.2, tar@^6.1.0, tar@^6.1.11, tar@^6.1.2: +tar@^6.0.2: + version "6.2.1" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" + integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^5.0.0" + minizlib "^2.1.1" + mkdirp "^1.0.3" + yallist "^4.0.0" + +tar@^6.1.0, tar@^6.1.11, tar@^6.1.2: version "6.2.0" resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.0.tgz#b14ce49a79cb1cd23bc9b016302dea5474493f73" integrity sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ== @@ -20680,6 +20746,15 @@ treeverse@^2.0.0: resolved "https://registry.yarnpkg.com/treeverse/-/treeverse-2.0.0.tgz#036dcef04bc3fd79a9b79a68d4da03e882d8a9ca" integrity sha512-N5gJCkLu1aXccpOTtqV6ddSEi6ZmGkh3hjmbu1IjcavJK4qyOVQmi0myQKM7z5jVGmD68SJoliaVrMmVObhj6A== +trigram-utils@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/trigram-utils/-/trigram-utils-1.0.3.tgz#535da37a414dae249c4b023512cf2b3dc65c8ea4" + integrity sha512-UAhS1Ll21FtClVIzIN0I/SmGnJ+D08BOxX7Dl1penV8raC0ksf2dJkhNI6kU1Mj3uT86Bul12iMvxXquXSYSng== + dependencies: + collapse-white-space "^1.0.0" + n-gram "^1.0.0" + trim "0.0.1" + trim-newlines@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" @@ -20690,6 +20765,11 @@ trim-newlines@^4.0.2: resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-4.0.2.tgz#d6aaaf6a0df1b4b536d183879a6b939489808c7c" integrity sha512-GJtWyq9InR/2HRiLZgpIKv+ufIKrVrvjQWEj7PxAXNc5dwbNJkqhAUoAGgzRmULAnoOM5EIpveYd3J2VeSAIew== +trim@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" + integrity sha512-YzQV+TZg4AxpKxaTHK3c3D+kRDCGVEE7LemdlQZoQXn0iennk10RsIoY6ikzAqJTc9Xjl9C1/waHom/J86ziAQ== + ts-algebra@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/ts-algebra/-/ts-algebra-1.2.0.tgz#f91c481207a770f0d14d055c376cbee040afdfc9" @@ -20973,10 +21053,10 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript@4.9.5, "typescript@^3 || ^4", typescript@^4.2.4, typescript@^5.3.3: - version "5.3.3" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.3.3.tgz#b3ce6ba258e72e6305ba66f5c9b452aaee3ffe37" - integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw== +"typescript@^3 || ^4", typescript@^4.2.4, typescript@^5.3.3: + version "5.4.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.4.2.tgz#0ae9cebcfae970718474fe0da2c090cad6577372" + integrity sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ== uWebSockets.js@uNetworking/uWebSockets.js#v20.34.0: version "20.34.0" @@ -21297,7 +21377,7 @@ v8-compile-cache-lib@^3.0.1: resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== -v8-compile-cache@2.3.0, v8-compile-cache@^2.0.3: +v8-compile-cache@2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== @@ -21755,7 +21835,7 @@ wildcard@^2.0.0: resolved "https://registry.yarnpkg.com/wildcard/-/wildcard-2.0.0.tgz#a77d20e5200c6faaac979e4b3aadc7b3dd7f8fec" integrity sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw== -word-wrap@^1.2.3, word-wrap@~1.2.3: +word-wrap@~1.2.3: version "1.2.4" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.4.tgz#cb4b50ec9aca570abd1f52f33cd45b6c61739a9f" integrity sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==