diff --git a/.gitignore b/.gitignore index d9bfb1db..e08d0e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ node_modules/ *.lock *.map .github/workflows/nodejs.yml - +go.work.sum app/**/*.map config/**/*.map diff --git a/.prettierrc.js b/.prettierrc.cjs similarity index 100% rename from .prettierrc.js rename to .prettierrc.cjs diff --git a/app/app.less b/app/app.less index 3a75536f..9a7b0df9 100644 --- a/app/app.less +++ b/app/app.less @@ -1,227 +1,237 @@ @import './common.less'; @font-face { - font-family: 'Roboto-Black'; - src: url('@app/static/fonts/Roboto-Black.ttf') format('truetype'); + font-family: 'Roboto-Black'; + src: url('@app/static/fonts/Roboto-Black.ttf') format('truetype'); } @font-face { - font-family: 'Roboto-Bold'; - src: url('@app/static/fonts/Roboto-Bold.ttf') format('truetype'); + font-family: 'Roboto-Bold'; + src: url('@app/static/fonts/Roboto-Bold.ttf') format('truetype'); } @font-face { - font-family: 'Roboto-Light'; - src: url('@app/static/fonts/Roboto-Light.ttf') format('truetype'); + font-family: 'Roboto-Light'; + src: url('@app/static/fonts/Roboto-Light.ttf') format('truetype'); } @font-face { - font-family: 'Roboto-Medium'; - src: url('@app/static/fonts/Roboto-Medium.ttf') format('truetype'); + font-family: 'Roboto-Medium'; + src: url('@app/static/fonts/Roboto-Medium.ttf') format('truetype'); } @font-face { - font-family: 'Roboto-Regular'; - src: url('@app/static/fonts/Roboto-Regular.ttf') format('truetype'); + font-family: 'Roboto-Regular'; + src: url('@app/static/fonts/Roboto-Regular.ttf') format('truetype'); } @font-face { - font-family: 'Roboto-Mono'; - src: url('@app/static/fonts/Roboto-Mono.ttf') format('truetype'); + font-family: 'Roboto-Mono'; + src: url('@app/static/fonts/Roboto-Mono.ttf') format('truetype'); } #studioApp { - font-family: Roboto-Regular, sans-serif; + font-family: Roboto-Regular, sans-serif; } .studioCenterLayout { - width: @containerWidth; - margin: 0 auto; + width: @containerWidth; + margin: 0 auto; } .studioTabHeader { - display: flex; - justify-content: center; - padding-bottom: 16px; + display: flex; + justify-content: center; + padding-bottom: 16px; } .ant-radio-group.studioTabGroup { - background: @lightGray; - border-radius: 20px; - padding: 4px; - min-width: 400px; - display: flex; - justify-content: center; - - .ant-radio-button-wrapper { - border-radius: 20px; - flex: 1; - text-align: center; - border: none; - white-space: nowrap; - color: @darkBlue; - - &.ant-radio-button-wrapper-checked { - color: #fff; - } - - &:not(.ant-radio-button-wrapper-checked) { - background: none; - } - - &::before { - width: 0; - } - } + background: @lightGray; + border-radius: 20px; + padding: 4px; + min-width: 400px; + display: flex; + justify-content: center; + + .ant-radio-button-wrapper { + border-radius: 20px; + flex: 1; + text-align: center; + border: none; + white-space: nowrap; + color: @darkBlue; + + &.ant-radio-button-wrapper-checked { + color: #fff; + } + + &:not(.ant-radio-button-wrapper-checked) { + background: none; + } + + &::before { + width: 0; + } + } } .ant-btn { - font-family: Roboto-Regular, sans-serif; + font-family: Roboto-Regular, sans-serif; } .ant-btn.warningBtn { - color: @red; - border-color: @red; - display: inline-flex; - align-items: center; - justify-content: center; - border-radius: 3px; + color: @red; + border-color: @red; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 3px; - svg { - width: 20px; - height: 20px; - } + svg { + width: 20px; + height: 20px; + } - &.ant-btn-link { - border: none; - } + &.ant-btn-link { + border: none; + } - &:hover { - color: @red; - border-color: @red; - } + &:hover { + color: @red; + border-color: @red; + } } .ant-btn.primaryBtn { - color: @blue; - border-color: @blue; - display: inline-flex; - align-items: center; - justify-content: center; - border-radius: 3px; + color: @blue; + border-color: @blue; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 3px; - svg { - width: 20px; - height: 20px; - } + svg { + width: 20px; + height: 20px; + } - &.ant-btn-link { - border: none; - } + &.ant-btn-link { + border: none; + } - &:hover { - color: @blue; - border-color: @blue; - } + &:hover { + color: @blue; + border-color: @blue; + } } .ant-btn.cancelBtn { - color: @darkGray; - border-color: @darkGray; - display: inline-flex; - align-items: center; - justify-content: center; - border-radius: 3px; + color: @darkGray; + border-color: @darkGray; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 3px; - svg { - width: 20px; - height: 20px; - } + svg { + width: 20px; + height: 20px; + } - &.ant-btn-link { - border: none; - } + &.ant-btn-link { + border: none; + } - &:hover { - color: @darkGray; - border-color: @darkGray; - } + &:hover { + color: @darkGray; + border-color: @darkGray; + } } .ant-btn.studioAddBtn { - border-radius: 3px; - display: inline-flex; - align-items: center; - padding: 0 29px; + border-radius: 3px; + display: inline-flex; + align-items: center; + padding: 0 29px; - a { - display: flex; - align-items: center; - } + a { + display: flex; + align-items: center; + } - .studioAddBtnIcon { - display: inline-flex; - margin-right: 10px; - height: 22px; + .studioAddBtnIcon { + display: inline-flex; + margin-right: 10px; + height: 22px; - > svg { - width: 22px; - height: 22px; - } + >svg { + width: 22px; + height: 22px; + } - & ~ span { - margin-left: 0; - } - } + &~span { + margin-left: 0; + } + } } .studioFormFooter { - position: fixed; - left: 0; - bottom: 0; - z-index: 10; - width: 100%; - height: 98px; - display: flex; - align-items: center; - justify-content: center; - background: #fff; - box-shadow: 0 -4px 4px rgba(0, 0, 0, 0.1); - - button { - width: 236px; - - &:not(:last-child) { - margin-right: 50px; - } - } + position: fixed; + left: 0; + bottom: 0; + z-index: 10; + width: 100%; + height: 98px; + display: flex; + align-items: center; + justify-content: center; + background: #fff; + box-shadow: 0 -4px 4px rgba(0, 0, 0, 0.1); + + button { + width: 236px; + + &:not(:last-child) { + margin-right: 50px; + } + } } .ant-form-item-label label { - font-family: Roboto-Bold, sans-serif; + font-family: Roboto-Bold, sans-serif; } .ant-table .ant-table-row { - background: #fff; + background: #fff; } // for thoes who use command pattern to create a Modal and other components // just `ConfigProvider` is not enough .ant-modal .ant-modal-content { - padding: 0; - .ant-modal-header { - padding: 16px 24px; - border-bottom: 1px solid #f0f0f0; - margin-bottom: 0; - } - .ant-modal-body { - padding: 24px; - .ant-modal-confirm-body-wrapper { - padding: 24px 0; - } - } - .ant-modal-footer { - padding: 10px 16px; - border-top: 1px solid #f0f0f0; - } + padding: 0; + + .ant-modal-header { + padding: 16px 24px; + border-bottom: 1px solid #f0f0f0; + margin-bottom: 0; + } + + .ant-modal-body { + padding: 24px; + + .ant-modal-confirm-body-wrapper { + padding: 24px 0; + } + } + + .ant-modal-footer { + padding: 10px 16px; + border-top: 1px solid #f0f0f0; + } } + +.ant-btn-default { + border: 1px solid @blue; + border-radius: 3px; + color: @blue; +} \ No newline at end of file diff --git a/app/components/FileConfigSetting/index.tsx b/app/components/FileConfigSetting/index.tsx index c995e861..24b17237 100644 --- a/app/components/FileConfigSetting/index.tsx +++ b/app/components/FileConfigSetting/index.tsx @@ -66,7 +66,7 @@ const FileConfigSetting = (props: IProps) => { const readFile = useCallback( debounce(() => { const { activeItem, setState } = state; - if (!activeItem) return; + if (!activeItem || !(activeItem.path.indexOf('.csv') > -1)) return; setState({ loading: true }); let content = []; if (activeItem.sample !== undefined) { @@ -93,7 +93,10 @@ const FileConfigSetting = (props: IProps) => { content = [...content, row.data]; }, complete: () => { - setState({ loading: false, previewContent: content }); + setState({ + loading: false, + previewContent: content, + }); }, }); } @@ -226,11 +229,12 @@ const FileConfigSetting = (props: IProps) => { ), key: 'withHeader', width: '30%', - render: (record) => ( - updateItem(e, record)}> - {intl.get('import.hasHeader')} - - ), + render: (record) => + record.path.indexOf('.csv') > -1 && ( + updateItem(e, record)}> + {intl.get('import.hasHeader')} + + ), }, { title: ( @@ -248,9 +252,10 @@ const FileConfigSetting = (props: IProps) => { ), key: 'delimiter', width: '30%', - render: (record) => ( - updateDelimiter(e, record)} /> - ), + render: (record) => + record.path.indexOf('.csv') > -1 && ( + updateDelimiter(e, record)} /> + ), }, { key: 'operation', diff --git a/app/config/locale/en-US.ts b/app/config/locale/en-US.ts index 7cecfcd0..5ff7f86f 100644 --- a/app/config/locale/en-US.ts +++ b/app/config/locale/en-US.ts @@ -69,6 +69,9 @@ export default { keyword: 'Keyword', function: 'Function', historyRecord: 'History', + language: 'Language', + switchOn: 'On', + switchOff: 'Off', }, doc: { welcome: 'Welcome to', @@ -168,6 +171,8 @@ export default { historyTip: 'You can directly use the "/" key in the console to quickly select historical query statements.', runSelectionRows: 'Run selected rows', selectEmpty: 'Please select the row', + send: 'Send', + copy2NGQL: 'Copy to Console', }, explore: { vertexStyle: 'Vertex Color', @@ -530,4 +535,32 @@ export default { alwaysShow: 'Always show the welcome page', progressTitle: 'Download & Import Data', }, + setting: { + globalSetting: 'Global Settings', + betaFunction: 'Beta Functions', + viewScmemaBetaFunDesc: + 'View the schema structure of the specified graph space, located at Schema-> Operations-> View Schema', + text2query: 'Text to Query', + text2queryDesc: 'You can use the LLM capability to write query statements in the console', + llmImport: 'AI Import', + llmImportDesc: 'Knowledge Graph Build with LLM and NebulaGraph, located at Import data -> New AI Import', + maxTextLength: 'Max Text Length', + verify: 'Verify', + }, + llm: { + newAiImport: 'New AI Import', + importTip: 'Please create the schema before creating an import task.', + setup: 'Setup', + confirm: 'Confirm', + file: 'File', + filePath: 'File Path', + importGraphSpace: 'Import Graph Space', + exportNGQLFilePath: 'Export NGQL File Path', + prompt: 'Prompt', + next: 'Next', + url: 'URL', + previous: 'Previous', + start: 'Start', + aiImport: 'AI Import', + }, }; diff --git a/app/config/locale/zh-CN.ts b/app/config/locale/zh-CN.ts index 40c5b090..7c55cf9f 100644 --- a/app/config/locale/zh-CN.ts +++ b/app/config/locale/zh-CN.ts @@ -69,6 +69,9 @@ export default { keyword: '关键字', function: '函数', historyRecord: '历史记录', + language: '语言', + switchOn: '开启', + switchOff: '关闭', }, doc: { welcome: '欢迎使用', @@ -162,6 +165,8 @@ export default { historyTip: '您可以直接输入“/”键快速选择历史查询语句。', runSelectionRows: '运行选中行', selectEmpty: '请先选择要运行的行', + send: '发送', + copy2NGQL: '复制到控制台', }, explore: { vertexStyle: '节点颜色', @@ -511,4 +516,30 @@ export default { alwaysShow: '始终展示欢迎页', progressTitle: '下载 & 导入数据', }, + setting: { + globalSetting: '全局设置', + betaFunction: 'Beta 功能', + viewScmemaBetaFunDesc: '查看指定图空间的 schema 结构，位于 Schema --> [图空间] --> 查看 Schema', + text2query: '文本转查询', + text2queryDesc: '在控制台中使用 LLM 功能，将自然语言转换为查询语句', + llmImport: 'AI 导入', + llmImportDesc: '使用 LLM 和 Nebula Graph 构建知识图谱，位于导入数据 --> 新建 AI 导入', + maxTextLength: '文本最大长度', + verify: '确认', + }, + llm: { + newAiImport: 'New AI Import', + importTip: '请先创建图空间和 schema', + setup: '设置', + confirm: '确认', + file: '文件', + filePath: '文件路径', + importGraphSpace: '导入图空间', + exportNGQLFilePath: '导出 NGQL 文件路径', + prompt: '提示', + next: '下一步', + previous: '上一步', + start: '开始', + aiImport: 'AI 导入', + }, }; diff --git a/app/interfaces/import.ts b/app/interfaces/import.ts index 6a5668ea..223079c4 100644 --- a/app/interfaces/import.ts +++ b/app/interfaces/import.ts @@ -1,3 +1,4 @@ +import { ILLMStatus } from '@app/pages/Import/TaskList/TaskItem/AIImportItem'; import { RcFile } from 'antd/lib/upload'; export enum ITaskStatus { @@ -36,6 +37,30 @@ export interface ITaskItem { message: string; stats: ITaskStats; rawConfig: string; + llmJob?: ILLMJob; +} + +export interface ILLMJob { + user_name: string; + host: string; + job_id: string; + space: string; + space_schema_string: string; + file?: string; + file_path?: string; + job_type: string; + status: ILLMStatus; + prompt_template: string; + process: { + total: number; + current: number; + ratio: number; + failed_reason: string; + prompt_tokens: number; + completion_tokens: number; + }; + update_time: string; + create_time: string; } export interface IPropertyProps { diff --git a/app/pages/Console/Setting.tsx b/app/pages/Console/Setting.tsx new file mode 100644 index 00000000..c64263e8 --- /dev/null +++ b/app/pages/Console/Setting.tsx @@ -0,0 +1,93 @@ +import { Radio, Form, Input, Modal, Checkbox, InputNumber, Switch } from 'antd'; +import { useEffect } from 'react'; +import { post } from '@app/utils/http'; +import llm from '@app/stores/llm'; +import { observer } from 'mobx-react-lite'; +import { useI18n } from '@vesoft-inc/i18n'; +import styles from './index.module.less'; + +function Setting({ open, setVisible }) { + const [form] = Form.useForm(); + const { intl } = useI18n(); + const onClose = () => { + setVisible(false); + }; + const onOk = async () => { + const values = await form.validateFields(); + const { url, key, llmVersion, apiType, ...config } = values; + const res = await post('/api/config/llm')({ + url, + key, + llmVersion, + apiType, + config: JSON.stringify(config), + }); + if (res.code === 0) { + setVisible(false); + llm.setConfig(values); + } + }; + + useEffect(() => { + if (!open) return; + initForm(); + }, [open]); + + async function initForm() { + await llm.fetchConfig(); + form.setFieldsValue(llm.config); + } + return ( + +
+ +
+
+ + + + LLM2nGQL + BETA +
+
+ + + + Copilot + BETA +
+
+ + + + + + + + + + + azure + openai + + + + + llm3.5-turbo + llm4 + + + + + {intl.get('console.useSpaceSchema')} + {intl.get('console.useConsoleNGQL')} + + + + + +
+ + ); +} +export default observer(Setting); diff --git a/app/pages/Import/AIImport/Create.tsx b/app/pages/Import/AIImport/Create.tsx new file mode 100644 index 00000000..f07a797d --- /dev/null +++ b/app/pages/Import/AIImport/Create.tsx @@ -0,0 +1,182 @@ +import { useStore } from '@app/stores'; +import { useI18n } from '@vesoft-inc/i18n'; +import { Button, Form, Input, Modal, Radio, Select, Tabs, message } from 'antd'; +import { observer } from 'mobx-react-lite'; +import styles from './index.module.less'; +import Icon from '@app/components/Icon'; +import { useEffect, useMemo, useState } from 'react'; +import { llmImportTask, llmImportPrompt } from '@app/stores/llm'; +import { getByteLength } from '@app/utils/function'; +import { post } from '@app/utils/http'; + +const Create = observer((props: { visible: boolean; onCancel: () => void }) => { + const { llm, schema, files } = useStore(); + const { fileList } = files; + const { intl } = useI18n(); + const [form] = Form.useForm(); + const [type, setType] = useState('file'); + const [step, setStep] = useState(0); + const [file, setFile] = useState(null); + const [space, setSpace] = useState(null); + const [tokens, setTokens] = useState(0); + const valuse = useMemo(() => { + return form.getFieldsValue(); + }, [step]); + useEffect(() => { + if (!props.visible) return; + llm.fetchConfig(); // refetch for update config + files.getFiles(); + setStep(0); + form.resetFields(); + form.setFieldsValue({ + type: 'file', + promptTemplate: llmImportPrompt, + }); + }, [props.visible]); + + const onNext = () => { + form.validateFields().then((values) => { + setStep(1); + }); + }; + + useEffect(() => { + (async () => { + if (file && space) { + const types = { + csv: 0.9, + json: 0.6, + pdf: 0.05, + }; + const subfix = file.name.split('.').pop(); + const type = types[subfix] || 0.5; + const size = file.size; + const schema = await llm.getSpaceSchema(space); + const schemaBytesLength = getByteLength(schema); + // full connection + const tokensNum = ((((schemaBytesLength * size) / 2000) * llm.config.maxContextLength) / 2000) * type; + setTokens(tokensNum); + } + })(); + }, [file, space]); + + const onConfirm = async () => { + const values = form.getFieldsValue(); + const schema = await llm.getSpaceSchema(space); + post('/api/llm/import/job')({ + type, + ...values, + spaceSchemaString: schema, + }).then((res) => { + if (res.code === 0) { + message.success(intl.get('common.success')); + props.onCancel(); + } + }); + }; + + return ( + +
+ + {intl.get('llm.importTip')} +
+
+
{ + setStep(0); + }} + > + + {intl.get('llm.setup')} +
+ +
+ + {intl.get('llm.confirm')} +
+
+ {tokens !== 0 && ( +
+ 🅣 prompt token: ~ + {Math.ceil(tokens / 10000)}w +
+ )} +
+ + {llm.config.features.includes('aiImportFilePath') && ( + + { + setType(e.target.value); + form.setFieldValue('file', undefined); + }} + > + {intl.get('llm.file')} + {intl.get('llm.filePath')} + + + )} + + {type === 'file' ? ( + + + + ) : ( + + + + )} + + + + + + + + + + +
+ +
+ + {type === 'file' ? valuse.file : valuse.filePath} + + + + +
+
+ + {step == 1 && } + {step == 0 && ( + + )} + {step == 1 && ( + + )} +
+ + ); +}); +export default Create; diff --git a/app/pages/Import/AIImport/index.module.less b/app/pages/Import/AIImport/index.module.less new file mode 100644 index 00000000..2a5b0096 --- /dev/null +++ b/app/pages/Import/AIImport/index.module.less @@ -0,0 +1,67 @@ +.buttonArea { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 10px; +} + +.tips { + display: flex; + padding: 10px 20px; + align-items: flex-start; + gap: 10px; + align-self: stretch; + background-color: #D4EBFF; + border-radius: 11px; + font-size: 16px; + font-style: normal; + font-weight: 500; + + svg { + color: #0D8BFF; + width: 24px; + height: 24px; + } +} + +.step { + display: flex; + align-items: center; + justify-content: center; + gap: 10px; + margin: 20px 0; + color: #0D8BFF; + + >div { + cursor: pointer; + display: flex; + align-items: center; + gap: 5px; + font-size: 16px; + + svg { + width: 35px; + height: 35px; + } + } + + >span { + + + width: 197px; + height: 1px; + display: inline-block; + border-bottom: 1px dashed #D4D4D8; + } +} + +.tokenNum { + position: absolute; + top: 200px; + right: 20px; + display: flex; + align-items: center; + gap: 5px; + justify-content: center; + color: #71717A; +} \ No newline at end of file diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx index b098b280..a326aa68 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx @@ -9,8 +9,8 @@ import { getFileSize } from '@app/utils/file'; import FileConfigSetting from '@app/components/FileConfigSetting'; import styles from './index.module.less'; type IUploadBtnProps = PropsWithChildren<{ - onUpload?: () => void -}> + onUpload?: () => void; +}>; const SizeLimit = 200 * 1024 * 1024; const UploadBtn = (props: IUploadBtnProps) => { @@ -21,12 +21,17 @@ const UploadBtn = (props: IUploadBtnProps) => { const { fileList, uploadFile } = files; const [visible, setVisible] = useState(false); const transformFile = async (_file: StudioFile, fileList: StudioFile[]) => { - const bigFiles = fileList.filter(file => file.size > SizeLimit); - if(bigFiles.length > 0) { - message.error(intl.get('import.fileSizeLimit', { name: bigFiles.map(i => i.name).join(', '), size: getFileSize(SizeLimit) })); + const bigFiles = fileList.filter((file) => file.size > SizeLimit); + if (bigFiles.length > 0) { + message.error( + intl.get('import.fileSizeLimit', { + name: bigFiles.map((i) => i.name).join(', '), + size: getFileSize(SizeLimit), + }), + ); return false; } - fileList.forEach(file => { + fileList.forEach((file) => { file.path = `${file.name}`; file.withHeader = false; file.delimiter = ','; @@ -46,7 +51,6 @@ const UploadBtn = (props: IUploadBtnProps) => { <> {}} @@ -63,11 +67,12 @@ const UploadBtn = (props: IUploadBtnProps) => { className={styles.uploadModal} footer={false} > - setVisible(false)} /> + onCancel={() => setVisible(false)} + /> ); diff --git a/app/pages/Import/TaskList/TaskItem/AIImportItem.tsx b/app/pages/Import/TaskList/TaskItem/AIImportItem.tsx new file mode 100644 index 00000000..32ed844d --- /dev/null +++ b/app/pages/Import/TaskList/TaskItem/AIImportItem.tsx @@ -0,0 +1,231 @@ +import { Button, Popconfirm, Progress, Tooltip, message as antMsg } from 'antd'; +import { CheckCircleFilled } from '@ant-design/icons'; +import { ITaskItem } from '@app/interfaces/import'; +import dayjs from 'dayjs'; +import { getFileSize } from '@app/utils/file'; +import Icon from '@app/components/Icon'; +import { useI18n } from '@vesoft-inc/i18n'; +import { observer } from 'mobx-react-lite'; +import { useStore } from '@app/stores'; +import styles from './index.module.less'; +import { _delete, post } from '@app/utils/http'; +import React from 'react'; +interface IProps { + data: ITaskItem; + onViewLog: (data: ITaskItem) => void; + onRefresh: () => void; +} +export enum ILLMStatus { + Running = 'running', + Success = 'success', + Failed = 'failed', + Cancel = 'cancel', + Pending = 'pending', +} +const llmStatusMap = { + [ILLMStatus.Running]: 'active', + [ILLMStatus.Success]: 'success', + [ILLMStatus.Failed]: 'execption', + [ILLMStatus.Cancel]: 'execption', + [ILLMStatus.Pending]: 'normal', +}; +const COLOR_MAP = { + success: { + from: '#8EDD3F', + to: '#27AE60', + }, + normal: { + from: '#8EDD3F', + to: '#27AE60', + }, + execption: { + from: '#EB5757', + to: '#EB5757', + }, + active: { + from: '#58D7FF', + to: '#2F80ED', + }, +}; +const loadingStatus = [ILLMStatus.Running, ILLMStatus.Pending]; +const AIImportItem = observer((props: IProps) => { + const { + data: { createTime, space, llmJob, id }, + onViewLog, + } = props; + const { intl } = useI18n(); + const { + dataImport: { downloadTaskConfig }, + moduleConfiguration, + } = useStore(); + const [rerunLoading, setRerunLoading] = React.useState(false); + const { disableConfigDownload } = moduleConfiguration.dataImport; + + const progressStatus = llmStatusMap[llmJob.status]; + + const onTaskDelete = () => { + _delete('/api/llm/import/job/' + llmJob.job_id)().then((res) => { + if (res.code === 0) { + antMsg.success(intl.get('import.deleteSuccess')); + props.onRefresh(); + } + }); + }; + + const onTaskStop = () => { + post('/api/llm/import/job/cancel')({ jobId: llmJob.job_id }).then((res) => { + if (res.code === 0) { + antMsg.success(intl.get('import.stopImportingSuccess')); + props.onRefresh(); + } + }); + }; + const handleRerun = async () => { + setRerunLoading(true); + const res = await post('/api/llm/import/job/rerun')({ jobId: llmJob.job_id }); + if (res.code === 0) { + antMsg.success(intl.get('common.rerunSuccess')); + props.onRefresh(); + } + setRerunLoading(false); + }; + + return ( + <> +
+
+ + {intl.get('common.space')}: {space} + +
+ {llmJob.status === ILLMStatus.Pending ? ( + <> + + {intl.get('import.modifyTime')}: {dayjs(llmJob.update_time).format('YYYY-MM-DD HH:mm:ss')} + + + ) : ( + <> + + {intl.get('common.createTime')}: {dayjs(createTime).format('YYYY-MM-DD HH:mm:ss')} + + + prompt tokens:{llmJob.process?.prompt_tokens || '-'}/ completion tokens:{' '} + {llmJob.process?.completion_tokens || '-'} + + + )} +
+
+
+
+
+ + + {intl.get('llm.aiImport')} + + {llmJob.space} + {llmJob.status === ILLMStatus.Success && ( + + + {intl.get('import.importCompleted')} + + {llmJob.process?.failed_reason && ` (${llmJob.process.failed_reason})`} + + + )} + {llmJob.status === ILLMStatus.Pending && ( + {intl.get('import.importPending')} + )} + {llmJob.status === ILLMStatus.Running && ( + {intl.get('import.importRunning')} + )} + {llmJob.status === ILLMStatus.Failed && ( + + {intl.get('import.importFailed')} + {llmJob.process?.failed_reason && ( + {` (${llmJob.process.failed_reason.slice( + 0, + 20, + )}...)`} + )} + + )} + {llmJob.status === ILLMStatus.Cancel && ( + {intl.get('import.importStopped')} + )} + +
+ {llmJob.process && ( + + {`${getFileSize(llmJob.process.current)} / `} + {getFileSize(llmJob.process.total)}{' '} + + )} +
+
+ `${percent.toFixed(1)}%`} + status={progressStatus as any} + percent={llmJob.process?.ratio * 100 || 0} + strokeColor={progressStatus && COLOR_MAP[progressStatus]} + /> +
+
+ + {llmJob.status === ILLMStatus.Running && ( + + + + )} + + {!loadingStatus.includes(llmJob.status) && ( + <> + + + + + + )} +
+
+
+ + ); +}); + +export default AIImportItem; diff --git a/app/pages/Import/TaskList/TaskItem/LogModal/index.module.less b/app/pages/Import/TaskList/TaskItem/LogModal/index.module.less index d1b38bd9..5eefa297 100644 --- a/app/pages/Import/TaskList/TaskItem/LogModal/index.module.less +++ b/app/pages/Import/TaskList/TaskItem/LogModal/index.module.less @@ -1,15 +1,18 @@ @import '@app/common.less'; + .logModal { height: 100vh; + :global { .ant-modal { height: 80%; + .ant-modal-content { height: 100%; } } } - + .importModalTitle { display: flex; align-items: center; @@ -25,22 +28,25 @@ border-bottom: none; padding-right: 80px; padding-top: 15px; + .ant-modal-title { display: flex; align-items: center; justify-content: space-between; } + .ant-modal-close { top: 5px; } } + .ant-modal-body { display: flex; height: 91%; } } } - + .logContainer { width: calc(100% - 200px); height: 100%; @@ -51,23 +57,34 @@ background: #333; color: #fff; white-space: nowrap; + display: flex; + flex-direction: column; + gap: 2px; } + .full { width: 100%; } - code { + + pre { font-size: 12px; font-family: inherit; + overflow: visible; + margin: 0; } } + .logTab { height: 100%; + :global { .ant-tabs-nav { width: 200px; + .ant-tabs-tab { background-color: @lightGray; color: @darkBlue; + .ant-tabs-tab-btn { word-break: break-all; width: 100%; @@ -75,20 +92,24 @@ text-align: left; } } + .ant-tabs-tab-active { background-color: @blue; color: #fff; + .ant-tabs-tab-btn { color: #fff; } } + .ant-tabs-nav-list { background-color: @lightGray; } } - .ant-tabs-content-holder > .ant-tabs-content { + + .ant-tabs-content-holder>.ant-tabs-content { display: none } - + } } \ No newline at end of file diff --git a/app/pages/Import/TaskList/TaskItem/LogModal/index.tsx b/app/pages/Import/TaskList/TaskItem/LogModal/index.tsx index 610c34c4..4c8e09c6 100644 --- a/app/pages/Import/TaskList/TaskItem/LogModal/index.tsx +++ b/app/pages/Import/TaskList/TaskItem/LogModal/index.tsx @@ -2,29 +2,20 @@ import { Button, Modal, Tabs } from 'antd'; import { useEffect, useRef, useState } from 'react'; import Icon from '@app/components/Icon'; import { useStore } from '@app/stores'; -import { ITaskStatus } from '@app/interfaces/import'; +import { ITaskItem, ITaskStatus } from '@app/interfaces/import'; import classnames from 'classnames'; import { useI18n } from '@vesoft-inc/i18n'; import styles from './index.module.less'; -interface ILogDimension { - space: string; - id: string; - status: ITaskStatus; -} - interface IProps { - logDimension: ILogDimension; + task: ITaskItem; visible: boolean; onCancel: () => void; } const LogModal = (props: IProps) => { - const { - visible, - onCancel, - logDimension: { space, id, status }, - } = props; + const { visible, onCancel, task } = props; + const { id, space, status, llmJob } = task; const { dataImport: { getLogs, downloadTaskLog, getLogDetail }, moduleConfiguration, @@ -35,24 +26,30 @@ const LogModal = (props: IProps) => { const [logs, setLogs] = useState([]); const [loading, setLoading] = useState(false); const [currentLog, setCurrentLog] = useState(null); + const [logData, setLogData] = useState([]); const handleTabChange = (key: string) => { setCurrentLog(logs.filter((item) => item === key)[0]); }; const getAllLogs = async () => { - const { code, data } = await getLogs(id); - if (code === 0) { - const logs = data.names || []; - setLogs(logs); - setCurrentLog(logs[0]); + if (!llmJob) { + const { code, data } = await getLogs(id); + if (code === 0) { + const logs = data.names || []; + setLogs(logs); + setCurrentLog(logs[0]); + } + } else { + setLogs(['all.log']); + setCurrentLog('all.log'); } }; const handleLogDownload = () => currentLog && downloadTaskLog({ id, name: currentLog }); const readLog = async () => { - const data = await getLogDetail({ id }); + const data = await getLogDetail(task); handleLogData(data); }; @@ -61,6 +58,7 @@ const LogModal = (props: IProps) => { if (!logs.length) { return; } + setLogData(logs.split('\n')); /** * {"level":"info",...} * {"level":"info",...} @@ -72,11 +70,11 @@ const LogModal = (props: IProps) => { * {"level":"info",...} * {"level":"info",...} */ - logRef.current.innerHTML = logs - .split('\n') - .map((log) => `${log}`) - .join('
'); - logRef.current.scrollTop = logRef.current.scrollHeight; + // logRef.current.innerHTML = logs + // .split('\n') + // .map((log) => `${log}`) + // .join('
'); + // logRef.current.scrollTop = logRef.current.scrollHeight; }; const initLog = async () => { @@ -128,7 +126,15 @@ const LogModal = (props: IProps) => { footer={false} > -
+
+ {logData.map((log, index) => { + return ( + 
+              {log}
+
+ ); + })} +
); }; diff --git a/app/pages/Import/TaskList/TaskItem/index.module.less b/app/pages/Import/TaskList/TaskItem/index.module.less index 791b9ea8..ac9fe398 100644 --- a/app/pages/Import/TaskList/TaskItem/index.module.less +++ b/app/pages/Import/TaskList/TaskItem/index.module.less @@ -1,4 +1,5 @@ @import '@app/common.less'; + .taskItem { background: #FFFFFF; box-shadow: 0px 2px 10px rgba(0, 0, 0, 0.15); @@ -6,6 +7,7 @@ min-height: 125px; margin-top: 20px; padding: 20px 30px; + .row { display: flex; justify-content: space-between; @@ -14,6 +16,7 @@ color: @darkGray; margin-bottom: 15px; align-items: center; + .createTime { font-weight: 600; font-size: 14px; @@ -21,63 +24,77 @@ font-family: Roboto-Light, serif; margin-right: 15px; } + :global(.ant-btn-link) { color: @darkGray; font-weight: 800; } } + .progress { flex: 1; margin-right: 30px; + .progressInfo { display: flex; justify-content: space-between; margin-bottom: 12px; padding-right: calc(2em + 8px); + .taskName { .draftLabel { background: #DBEFFF; border-radius: 4px; padding: 6px 12px; - font-weight: 500; font-size: 14px; color: @darkBlue; margin-right: 10px; } - font-family: Roboto-Bold, serif; + + font-family: Roboto-Bold, + serif; font-style: normal; font-size: 14px; color: @lightBlack; - & > span:not(.draftLabel) { + + &>span:not(.draftLabel) { margin-left: 12px; } + .completeInfo { color: #27AE60; - & > span { + + &>span { margin-right: 6px } + .red { color: @red } } + .errInfo { color: @red } - } + } + .moreInfo { - & > span:not(:last-child) { + &>span:not(:last-child) { margin-right: 30px; } } } + :global(.ant-progress-text) { color: @lightBlack; } } + .operations { :global { .ant-btn { height: 38px; + &:not(:last-child) { margin-right: 15px; } diff --git a/app/pages/Import/TaskList/TaskItem/index.tsx b/app/pages/Import/TaskList/TaskItem/index.tsx index 1155726c..8e697ac4 100644 --- a/app/pages/Import/TaskList/TaskItem/index.tsx +++ b/app/pages/Import/TaskList/TaskItem/index.tsx @@ -17,7 +17,7 @@ interface IProps { data: ITaskItem; onTaskStop: (id: string) => void; onTaskDelete: (id: string) => void; - onViewLog: (id: string, space: string, status: ITaskStatus) => void; + onViewLog: (task: ITaskItem) => void; onRerun: () => void; } @@ -236,7 +236,7 @@ const TaskItem = (props: IProps) => { {!isDraft && !loadingStatus.includes(status) && ( - )} + {global.appSetting.beta.functions.llmImport && ( + + )}
+ + ); + } + }); + } + + return ( +
{ + e.preventDefault(); + e.stopPropagation(); + }} + > +
+
+ {messages.map((item, index) => { + return ( +
+
+
{renderContent(item)}
+
+
+ ); + })} +
+
+
+ { + llm.update({ + currentInput: e.target.value, + }); + }} + /> + +
+
+ ); +} + +export default observer(Chat); diff --git a/app/pages/LLMBot/index.module.less b/app/pages/LLMBot/index.module.less new file mode 100644 index 00000000..189a9e81 --- /dev/null +++ b/app/pages/LLMBot/index.module.less @@ -0,0 +1,64 @@ +.llmBot { + position: fixed; + bottom: 20px; + right: 20px; + font-size: 5px; + cursor: pointer; +} + +.llmBotTitle { + font-size: 16px; + + .llmBotHandler { + font-size: 14px; + color: #828282; + } +} + +@size: 60px; + +.ball { + transition: all 0.3s; + position: relative; + border-radius: 50%; + width: @size; + height: @size; + background: transparent; + backdrop-filter: blur(3px); + box-shadow: 0 1px 10px 1px rgba(0, 0, 0, 0.3); + display: flex; + align-items: center; + justify-content: center; + + svg { + font-size: 40px; + } + + .open svg { + fill: url(#llm-icon); + } +} + +:global { + #icon-studio-btn-consoleGTP { + fill: url(#llm-icon) !important; + } +} + + +@keyframes inner-roll { + 0% { + transform: rotate3d(0, 1, 1, 90deg) rotateZ(-572.4deg) + } + + 100% { + transform: rotate3d(1, 0, 0 90deg) rotateZ(572.4deg) + } +} + +.llmBotTitle { + display: flex; + justify-content: space-between; + align-items: center; + width: 100%; +} \ No newline at end of file diff --git a/app/pages/LLMBot/index.tsx b/app/pages/LLMBot/index.tsx new file mode 100644 index 00000000..56ba6139 --- /dev/null +++ b/app/pages/LLMBot/index.tsx @@ -0,0 +1,68 @@ +import { Popover, Switch } from 'antd'; +import styles from './index.module.less'; +import Chat from './chat'; +import Icon from '@app/components/Icon'; +import { observer } from 'mobx-react-lite'; +import rootStore, { useStore } from '@app/stores'; +// float llm bot window +function LLMBot() { + const { global, llm } = useStore(); + if (global.appSetting?.beta?.functions?.text2query.open != true) { + return null; + } + const { open } = llm; + + return ( + <> + { + llm.update({ + open: visible, + }); + }} + content={} + title={ +
+
AI Asistant
+
+ text2match + { + llm.update({ + mode: checked ? 'text2cypher' : 'text2ngql', + }); + }} + checked={llm.mode == 'text2cypher'} + /> +
+
+ } + trigger={'click'} + > +
+
+ {!open ? ( + + ) : ( + + )} +
+
+ + + + + + + + + + + ); +} + +export default observer(LLMBot); diff --git a/app/pages/MainPage/Header/HelpMenu/index.tsx b/app/pages/MainPage/Header/HelpMenu/index.tsx index 2245ec8d..7f6d5adf 100644 --- a/app/pages/MainPage/Header/HelpMenu/index.tsx +++ b/app/pages/MainPage/Header/HelpMenu/index.tsx @@ -127,6 +127,14 @@ const HelpMenu = () => { }, ], }, + { + key: 'setting', + label: ( + + + + ), + }, { key: 'user', popupClassName: styles.accountMenu, diff --git a/app/pages/MainPage/Header/index.tsx b/app/pages/MainPage/Header/index.tsx index eebbd254..fb9fec81 100644 --- a/app/pages/MainPage/Header/index.tsx +++ b/app/pages/MainPage/Header/index.tsx @@ -17,60 +17,73 @@ interface IMenuItem { category: string; action: string; label: string; - }, - icon: string, - intlKey: string + }; + icon: string; + intlKey: string; } interface IProps { - menus: IMenuItem[] + menus: IMenuItem[]; } const PageHeader = (props: IProps) => { const { menus } = props; const { intl, currentLocale } = useI18n(); const [activeKey, setActiveKey] = useState(''); - const { global: { username, host } } = useStore(); + const { + global: { username, host }, + } = useStore(); const { pathname } = useLocation(); const handleMenuClick = async ({ key }) => { setActiveKey(key); }; - const MenuItems = useMemo(() => menus.map((item) => ({ - key: item.key, - label: - - {intl.get(item.intlKey)} - - })), [currentLocale]); + const MenuItems = useMemo( + () => + menus.map((item) => ({ + key: item.key, + label: ( + + + {intl.get(item.intlKey)} + + ), + })), + [currentLocale], + ); useEffect(() => { const activeKey = pathname.split('/')[1]; setActiveKey(activeKey); }, [pathname]); - return
- - logo - - {host && username ? <> - - - - : } -
; + return ( +
+ + logo + + {host && username ? ( + <> + + + + ) : ( + + )} +
+ ); }; export default observer(PageHeader); diff --git a/app/pages/MainPage/index.tsx b/app/pages/MainPage/index.tsx index 2773059e..11d381ee 100644 --- a/app/pages/MainPage/index.tsx +++ b/app/pages/MainPage/index.tsx @@ -4,6 +4,7 @@ import { Redirect, Route, Switch } from 'react-router-dom'; import { shouldAlwaysShowWelcome } from '@app/pages/Welcome'; import ErrorBoundary from '@app/components/ErrorBoundary'; import { MENU_LIST, RoutesList } from './routes'; +import LLMBot from '../LLMBot'; import './index.less'; import Header from './Header'; @@ -35,6 +36,7 @@ const MainPage = () => { + ); }; diff --git a/app/pages/MainPage/routes.tsx b/app/pages/MainPage/routes.tsx index 86a1ec63..02f00cd8 100644 --- a/app/pages/MainPage/routes.tsx +++ b/app/pages/MainPage/routes.tsx @@ -1,4 +1,5 @@ import { lazy } from 'react'; +import Setting from '../Setting'; const Schema = lazy(() => import('@app/pages/Schema')); const Console = lazy(() => import('@app/pages/Console')); @@ -9,7 +10,6 @@ const TaskCreate = lazy(() => import('@app/pages/Import/TaskCreate')); const SketchModeling = lazy(() => import('@app/pages/SketchModeling')); const Welcome = lazy(() => import('@app/pages/Welcome')); - export const RoutesList = [ { path: '/schema', @@ -54,6 +54,11 @@ export const RoutesList = [ component: Welcome, exact: true, }, + { + path: '/setting', + component: Setting, + exact: true, + }, ]; export const MENU_LIST = [ @@ -63,10 +68,10 @@ export const MENU_LIST = [ track: { category: 'navigation', action: 'view_schema', - label: 'from_navigation' + label: 'from_navigation', }, icon: 'icon-studio-nav-schema', - intlKey: 'common.schema' + intlKey: 'common.schema', }, { key: 'import', @@ -74,10 +79,10 @@ export const MENU_LIST = [ track: { category: 'navigation', action: 'view_import', - label: 'from_navigation' + label: 'from_navigation', }, icon: 'icon-studio-nav-import', - intlKey: 'common.import' + intlKey: 'common.import', }, { key: 'console', @@ -85,10 +90,10 @@ export const MENU_LIST = [ track: { category: 'navigation', action: 'view_console', - label: 'from_navigation' + label: 'from_navigation', }, icon: 'icon-studio-nav-console', - intlKey: 'common.console' + intlKey: 'common.console', }, { key: 'sketch', @@ -96,9 +101,9 @@ export const MENU_LIST = [ track: { category: 'navigation', action: 'view_sketch', - label: 'from_navigation' + label: 'from_navigation', }, icon: 'icon-navbar-sketch', - intlKey: 'common.sketch' + intlKey: 'common.sketch', }, -]; \ No newline at end of file +]; diff --git a/app/pages/Schema/SchemaConfig/Create/IndexCreate/index.module.less b/app/pages/Schema/SchemaConfig/Create/IndexCreate/index.module.less index bd7a198d..1d2f7a3d 100644 --- a/app/pages/Schema/SchemaConfig/Create/IndexCreate/index.module.less +++ b/app/pages/Schema/SchemaConfig/Create/IndexCreate/index.module.less @@ -1,13 +1,17 @@ @import '@app/common.less'; + .indexCreatePage { padding-bottom: 100px; + .btnFieldAdd { margin-bottom: 20px; } + .viewRow { padding-top: 24px; border-top: 1px solid @gray; } + .dragItem { display: flex; align-items: center; @@ -20,9 +24,11 @@ font-size: 14px; padding: 12px 12px 12px 16px; margin-bottom: 5px; + :global(.ant-tag-close-icon) { margin-left: 10px; color: @darkBlue; + svg { width: 14px; height: 14px; @@ -36,16 +42,18 @@ .ant-modal-content .ant-modal-header { border-bottom: none; } + .ant-modal-content .ant-modal-body { text-align: center; } } - + .modalItem { display: flex; align-items: center; justify-content: center; margin-bottom: 20px; + :global(.studioIconInstruction) { svg { width: 23px; @@ -59,10 +67,12 @@ width: 330px; border: 1px solid @gray; border-radius: 3px; + :global { .ant-select-selector { border: none; } + .ant-select-arrow svg { color: @darkBlue; } @@ -78,9 +88,11 @@ .ant-modal-footer { border-top: none; text-align: center; + .ant-btn { width: 140px; } + .ant-btn-default { border: 1px solid @darkBlue; border-radius: 3px; diff --git a/app/pages/Setting/index.module.less b/app/pages/Setting/index.module.less new file mode 100644 index 00000000..d559bf9a --- /dev/null +++ b/app/pages/Setting/index.module.less @@ -0,0 +1,150 @@ +@import '@app/common.less'; + +.tips { + color: #828282; + font-size: 12px; +} + +.settingPage { + width: @containerWidth; + margin: 0 auto; + padding: 25px 0; + + & :global { + .ant-card { + &:not(:last-child) { + margin-bottom: 20px; + } + + .ant-card-head { + padding: 0 16px; + + .ant-card-head-title { + padding: 10px 0; + } + } + + .ant-card-body { + padding: 16px; + } + } + } +} + +.pageTitle { + font-family: Roboto-Regular; + font-weight: bold; + font-size: 18px; + padding-bottom: 12px; + border-bottom: 1px solid @gray; +} + +.pageContent { + .cardTitle { + display: flex; + align-items: center; + } + + .settingItem { + width: 980px; + margin: auto; + border-bottom: 1px solid #d5ddeb; + padding: 30px 0; + + .title { + width: 200px; + display: flex; + } + } + + .betaFunItem { + display: flex; + + &:not(:last-child) { + padding-bottom: 12px; + margin-bottom: 12px; + border-bottom: 1px solid #f5f5f5; + } + + .itemContent { + margin-left: 12px; + + .betaFunName { + margin-right: 16px; + // flex: 0 0 100px; + } + } + + .tips { + margin-top: 6px; + } + } + + .sliderWrapper { + display: flex; + align-items: center; + font-size: 14px; + + :global { + .ant-slider { + .ant-slider-rail { + height: 6px; + } + + .ant-slider-track { + height: 6px; + } + + .ant-slider-step { + height: 6px; + } + + .ant-slider-handle { + width: 16px; + height: 16px; + } + } + + .ant-input-number-input { + height: 30px; + } + } + } + + & :global { + .ant-table-thead>tr>th { + background-color: #fafafa; + } + } + + .imageSwitcher { + display: flex; + flex-direction: column; + + .imageWrapper { + width: 320px; + overflow: hidden; + background-color: #2f3a4a; + + img { + width: 100%; + height: 100%; + object-fit: cover; + } + } + + .avatarUploader { + margin-top: 12px; + + .uploadBtn { + text-align: center; + padding: 2px 12px; + height: 32px; + } + + .tips { + margin-left: 12px; + } + } + } +} \ No newline at end of file diff --git a/app/pages/Setting/index.tsx b/app/pages/Setting/index.tsx new file mode 100644 index 00000000..c645aac0 --- /dev/null +++ b/app/pages/Setting/index.tsx @@ -0,0 +1,151 @@ +import { useCallback, useEffect, useState } from 'react'; +import { observer } from 'mobx-react-lite'; +import { Button, Col, Form, Input, InputNumber, Row, Select, Switch, message } from 'antd'; +import { useI18n } from '@vesoft-inc/i18n'; +import { useStore } from '@app/stores'; +// import LanguageSelect from '@app/components/LanguageSelect'; +import { trackEvent } from '@app/utils/stat'; +import styles from './index.module.less'; +import LanguageSelect from '../Login/LanguageSelect'; +import { useForm } from 'antd/lib/form/Form'; +import { post } from '@app/utils/http'; + +const Setting = observer(() => { + const { intl } = useI18n(); + const { global, llm } = useStore(); + const { appSetting, saveAppSetting } = global; + const [form] = useForm(); + const [apiType, setApiType] = useState('openai'); + useEffect(() => { + initForm(); + }, []); + + const initForm = async () => { + await llm.fetchConfig(); + form.setFieldsValue(llm.config); + setApiType(llm.config.apiType); + }; + + const updateAppSetting = useCallback(async (param: Partial) => { + // make it loading for a while, so it looks more smooth + saveAppSetting({ beta: { ...global?.appSetting?.beta, ...param } }); + trackEvent('setting', 'update_app_setting'); + }, []); + + const onSubmitLLMForm = useCallback(() => { + form.validateFields().then((values) => { + const { apiType, url, key, maxContextLength, ...config } = values; + post('/api/config/llm')({ + apiType, + url, + key, + maxContextLength, + config: JSON.stringify(config), + }).then((res) => { + if (res.code === 0) { + message.success(intl.get('common.success')); + llm.setConfig(values); + } + }); + }); + }, [form]); + + const { open = true, functions } = appSetting?.beta || {}; + + return ( +
+
{intl.get('setting.globalSetting')}
+
+ + + {intl.get('common.language')} + + + + + + + + {intl.get('setting.betaFunction')} +
+ updateAppSetting({ open: checked })} + /> +
+ + +
+ updateAppSetting({ functions: { ...functions, viewSchema: { open } } })} + /> +
+
{intl.get('common.viewSchema')}
+
{intl.get('setting.viewScmemaBetaFunDesc')}
+
+
+
+ updateAppSetting({ functions: { ...functions, text2query: { open } } })} + /> +
+
{intl.get('setting.text2query')}
+
{intl.get('setting.text2queryDesc')}
+
+
+
+ updateAppSetting({ functions: { ...functions, llmImport: { open } } })} + /> +
+
{intl.get('setting.llmImport')}
+
{intl.get('setting.llmImportDesc')}
+
+ + + + + + + + + + {apiType === 'qwen' && ( + + + + )} + + + + + + +
+
+
+ + +
+
+ ); +}); +export default Setting; diff --git a/app/stores/global.ts b/app/stores/global.ts index cb7bfaff..a5df3317 100644 --- a/app/stores/global.ts +++ b/app/stores/global.ts @@ -4,15 +4,18 @@ import { Base64 } from 'js-base64'; import { BrowserHistory } from 'history'; import service from '@app/config/service'; import ngqlRunner from '@app/utils/websocket'; -import { isValidIP } from '@app/utils/function'; +import { isValidIP, safeParse } from '@app/utils/function'; import { getRootStore, resetStore } from '.'; export class GlobalStore { gConfig = window.gConfig; appSetting = { beta: { + open: true, functions: { viewSchema: { open: true }, + text2query: { open: true }, + llmImport: { open: true }, }, }, }; @@ -28,11 +31,22 @@ export class GlobalStore { _username: observable, _host: observable, ngqlRunner: observable.ref, + appSetting: observable.ref, update: action, + saveAppSetting: action, }); this.ngqlRunner.logoutFun = this.logout; + const cacheAppSetting = localStorage.getItem('appSetting'); + if (cacheAppSetting) { + this.appSetting = safeParse(cacheAppSetting); + } } + saveAppSetting = (payload: any) => { + this.appSetting = payload; + localStorage.setItem('appSetting', JSON.stringify(payload)); + }; + get rootStore() { return getRootStore(); } @@ -44,6 +58,8 @@ export class GlobalStore { return this._host || cookies.get('nh'); } + useLocalObservable; + resetModel = () => { this.update({ _username: '', diff --git a/app/stores/import.ts b/app/stores/import.ts index 3a3c802d..14f092a1 100644 --- a/app/stores/import.ts +++ b/app/stores/import.ts @@ -341,7 +341,7 @@ export class ImportStore { trackEvent('import', 'download_task_log'); }; - getLogDetail = async (params: { id: string }) => { + getLogDetail = async (params: ITaskItem) => { const { code, data } = await service.getLogDetail(params); if (code === 0) { return data; diff --git a/app/stores/index.ts b/app/stores/index.ts index 831c7af2..d39a0433 100644 --- a/app/stores/index.ts +++ b/app/stores/index.ts @@ -9,6 +9,7 @@ import sketchModel from './sketchModel'; import welcome from './welcome'; import datasource from './datasource'; import moduleConfiguration from './moduleConfiguration'; +import llm from './llm'; const rootStore = { global, @@ -21,6 +22,7 @@ const rootStore = { welcome, datasource, moduleConfiguration, + llm, }; const rootStoreRef = { current: rootStore }; // @ts-ignore diff --git a/app/stores/llm.ts b/app/stores/llm.ts new file mode 100644 index 00000000..93fa73e2 --- /dev/null +++ b/app/stores/llm.ts @@ -0,0 +1,292 @@ +import { makeAutoObservable } from 'mobx'; +import schema from './schema'; +import { get } from '@app/utils/http'; +import rootStore from '.'; +import ws from '@app/utils/websocket'; +import { safeParse } from '@app/utils/function'; +import * as ngqlDoc from '@app/utils/ngql'; + +export const matchPrompt = `Use NebulaGraph match knowledge to help me answer question. +Use only the provided relationship types and properties in the schema. +Do not use any other relationship types or properties that are not provided. +Schema: +--- +{schema} +--- +Note: NebulaGraph speaks a dialect of Cypher, comparing to standard Cypher: +1. it uses double equals sign for comparison: == rather than = +2. it needs explicit label specification when referring to node properties, i.e. +v is a variable of a node, and we know its label is Foo, v.foo.name is correct +while v.name is not. +For example, see this diff between standard and NebulaGraph Cypher dialect: +diff +< MATCH (p:person)-[:directed]->(m:movie) WHERE m.name = 'The Godfather' +< RETURN p.name; +--- +> MATCH (p:person)-[:directed]->(m:movie) WHERE m.movie.name == 'The Godfather' +> RETURN p.person.name; +Question:{query_str} +`; +export const llmImportPrompt = `You are knowledge graph Expert. +please extract relationship data from the text below, referring to the schema of the graph, and return the results in the following JSON format without interpreting, don't explain just return the results directly. +{ + "nodes":[{ "name":"","type":"","props":{} }], + "edges":[{ "src":"","dst":"","edgeType":"","props":{} }] +} +The schema of the graph is: {spaceSchema} +The text is: {text} +The result is: +`; +export const llmImportTask = `please excute the task below,and return the result,dont' explain,just return the result directly. +{ + "task": "extract relationships", + "instructions": { + "text": "{text}", + "graphSchema": "{spaceSchema}", + "format": { + "nodes": [{ + "name": "", + "type": "", + "props": {} + }], + "edges": [{ + "src": "", + "dst": "", + "type": "", + "props": "{props}" + }] + } + } +} +reuslt:`; + +export interface LLMConfig { + url: string; + apiType: string; + llmVersion: string; + key: string; + features: string[]; + maxContextLength: number; + enableCopilot: boolean; + enableLLM2NGQLs: boolean; + gqlPath: string; +} +class LLM { + currentInput = ''; + open = false; + config = { + maxContextLength: 2000, + url: 'https://{your-resource-name}.openai.azure.com/openai/deployments/{deployment-id}/chat/completions?api-version={api-version}', + apiType: 'openai', + features: ['spaceSchema', 'useConsoleNGQL'], + } as LLMConfig; + widget: HTMLSpanElement; + editor: any; + mode = 'text2ngql' as 'text2ngql' | 'text2cypher'; + completionList: { text: string; type: string }[] = []; + constructor() { + makeAutoObservable(this, { + editor: false, + widget: false, + }); + this.fetchConfig(); + } + + fetchConfig() { + return get('/api/config/llm')().then((res) => { + if (res.code != 0 || !res.data) return; + const { config, ...values } = res.data.config; + const configMap = config ? safeParse(config) : {}; + this.setConfig({ + ...configMap, + ...values, + gqlPath: res.data.gqlPath, + }); + return this.config; + }); + } + + setConfig(payload: LLMConfig) { + this.config = { ...this.config, ...payload }; + } + + update(payload: any) { + Object.assign(this, payload); + } + + async getSpaceSchema(space: string) { + let finalPrompt: any = { + spaceName: space, + }; + if (this.config.features.includes('spaceSchema')) { + await schema.switchSpace(space); + await schema.getTagList(); + await schema.getEdgeList(); + const tagList = schema.tagList; + const edgeList = schema.edgeList; + finalPrompt = { + ...finalPrompt, + vidType: schema.spaceVidType, + nodeTypes: tagList.map((item) => { + return { + type: item.name, + props: item.fields.map((item) => { + return { + name: item.Field, + dataType: item.Type, + nullable: (item as any).Null === 'YES', + }; + }), + }; + }), + edgeTypes: edgeList.map((item) => { + return { + type: item.name, + props: item.fields.map((item) => { + return { + name: item.Field, + dataType: item.Type, + nullable: (item as any).Null === 'YES', + }; + }), + }; + }), + }; + } + return JSON.stringify(finalPrompt); + } + + async getDocPrompt(text: string) { + let prompt = matchPrompt; // default use text2cypher + if (this.mode !== 'text2cypher') { + text = text.replaceAll('"', "'"); + const res = (await ws.runChat({ + req: { + temperature: 0.5, + stream: false, + max_tokens: 20, + messages: [ + { + role: 'user', + content: `From the following graph database book categories: "${ngqlDoc.NGQLCategoryString}" find top two useful categories to solve the question:"${text}",don't explain,just return the two combined categories, separated by ',' is:`, + }, + ], + }, + })) as any; + if (res.code === 0) { + const url = res.message.choices[0].message?.content as string; + const paths = url + .toLowerCase() + .replaceAll(/\s|"|\\/g, '') + .split(','); + console.log('select doc url:', paths); + if (ngqlDoc.ngqlMap[paths[0]]) { + let doc = ngqlDoc.ngqlMap[paths[0]].content; + if (!doc) { + doc = ''; + } + const doc2 = ngqlDoc.ngqlMap[paths[1]].content; + if (doc2) { + doc += doc2; + } + doc = doc.replaceAll(/\n\n+/g, ''); + if (doc.length) { + console.log('docString:', doc); + prompt = `learn the below doc, and use it to help user ,the user space schema is "{schema}" the doc is: \n${doc.slice( + 0, + this.config.maxContextLength, + )} the question is "{query_str}"`; + } + } + } + } + prompt = prompt.replace('{query_str}', text); + const pathname = window.location.pathname; + const space = pathname.indexOf('schema') > -1 ? rootStore.schema.currentSpace : rootStore.console.currentSpace; + if (!space) { + return prompt.replace('{schema}', 'no space selected'); + } + let schemaPrompt = await this.getSpaceSchema(space); + + if (this.config.features.includes('useConsoleNGQL')) { + schemaPrompt += `\nuser console ngql context: ${rootStore.console.currentGQL}`; + } + prompt = prompt.replace('{schema}', schemaPrompt); + return prompt; + } + + timer; + running = false; + async checkCopilotList(cm: any) { + clearTimeout(this.timer); + this.timer = setTimeout(async () => { + let snippet = ''; + const cursor = cm.getCursor(); + const line = cm.getLine(cursor.line).split(';').pop(); + if (cursor.ch < line.length - 1) return; + if (line.length < 3) return; + const tokens = line.split(' '); + const firstToken = tokens.find((item) => item.replaceAll(' ', '').length > 0); + const hits = ngqlDoc.ngqlDoc.filter((each) => each.title.toLowerCase().indexOf(firstToken.toLowerCase()) === 0); + let doc = ''; + if (this.mode == 'text2cypher' && firstToken.toLowerCase() == 'match') { + doc += matchPrompt; + } else { + if (hits.length) { + hits.find((item) => { + if (doc.length > this.config.maxContextLength) return true; + doc += item + '\n'; + }); + } + } + if (!doc) { + return; + } + this.running = true; + cm.closeHint(); + const schema = await this.getSpaceSchema(rootStore.console.currentSpace); + const res = (await ws.runChat({ + req: { + temperature: 1.0, + stream: false, + presence_penalty: 0.6, + max_tokens: 30, + messages: [ + { + role: 'user', + content: `As a NebulaGraph NGQL code autocomplete copilot, you have access to the following information: document "${doc}" and user space schema "${schema}". + Use this information to guess the user's next NGQL code autocomplete as accurately as possible. + Please provide your guess as a response without any prefix words. + Don't explain anything. + the next autocomplete text can combine with the given text. + if you can't guess, say "Sorry", + The user's NGQL text is: ${line} + the most possible 2 next autocomplete text is:`, + }, + ], + }, + })) as any; + if (res.code === 0) { + snippet = res.message.choices[0].message?.content; + console.log(snippet); + if (snippet.indexOf('Sorry') > -1) { + snippet = ''; + } + } + if (snippet) { + this.update({ + completionList: snippet + .split('\n') + .map((each) => ({ + type: 'copilot', + text: each, + })) + .filter((item) => item.text !== ''), + }); + } + this.running = false; + }, 3000); + } +} + +export default new LLM(); diff --git a/app/stores/schema.ts b/app/stores/schema.ts index 76357198..43f8802e 100644 --- a/app/stores/schema.ts +++ b/app/stores/schema.ts @@ -270,7 +270,10 @@ export class SchemaStore { }; const { code, data } = await this.getTagOrEdgeInfo(ISchemaEnum.Edge, item, space); if (code === 0) { - edge.fields = data.tables; + edge.fields = data.tables.map((item) => ({ + Field: item.Field, + Type: item.Type, + })); } edgeList.push(edge); }), diff --git a/app/utils/file.ts b/app/utils/file.ts index 27fe56ac..a208cdd6 100644 --- a/app/utils/file.ts +++ b/app/utils/file.ts @@ -10,11 +10,14 @@ export function readFileContent(file) { export function getFileSize(size: number) { const units = ['B', 'KB', 'MB', 'GB', 'TB'] as const; const gap = 1 << 10; + if (!size) { + return '0 B'; + } for (let i = 0, byte = gap; i < units.length; i++, byte *= gap) { if (size < byte || i === units.length - 1) { const unitSize = ((size * gap) / byte).toFixed(2); - return `${unitSize} ${units[i]}` as `${number} ${typeof units[number]}`; + return `${unitSize} ${units[i]}` as `${number} ${(typeof units)[number]}`; } } } diff --git a/app/utils/ngql.json b/app/utils/ngql.json new file mode 100644 index 00000000..99ae15e4 --- /dev/null +++ b/app/utils/ngql.json @@ -0,0 +1 @@ +[{"title":"Welcome to NebulaGraph 3.6.0 Documentation","content":"NebulaGraph is a distributed, scalable, and lightning-fast graph database. It is the optimal solution in the world capable of hosting graphs with dozens of billions of vertices (nodes) and trillions of edges (relationships) with millisecond latency.\nGetting started\nQuick start\nPreparations before deployment\nnGQL cheatsheet\nFAQ\nEcosystem Tools\nLive Demo\nRelease notes\nNebulaGraph Community Edition 3.6.0\nNebulaGraph Dashboard Community\nNebulaGraph Studio\nOther Sources\nTo cite NebulaGraph\nForum\nNebulaGraph Homepage\nBlogs\nVideos\nChinese Docs\nSymbols used in this manual\nModify errors\nThis NebulaGraph manual is written in the Markdown language. Users can click the pencil sign on the upper right side of each document title and modify errors.","url":"https://docs.nebula-graph.io/3.6.0/.","type":"doc"},{"title":"What is NebulaGraph","content":"NebulaGraph is an open-source, distributed, easily scalable, and native graph database. It is capable of hosting graphs with hundreds of billions of vertices and trillions of edges, and serving queries with millisecond-latency. \nWhat is a graph database\nA graph database, such as NebulaGraph, is a database that specializes in storing vast graph networks and retrieving information from them. It efficiently stores data as vertices (nodes) and edges (relationships) in labeled property graphs. Properties can be attached to both vertices and edges. Each vertex can have one or multiple tags (labels).\nGraph databases are well suited for storing most kinds of data models abstracted from reality. Things are connected in almost all fields in the world. Modeling systems like relational databases extract the relationships between entities and squeeze them into table columns alone, with their types and properties stored in other columns or even other tables. This makes data management time-consuming and cost-ineffective.\nNebulaGraph, as a typical native graph database, allows you to store the rich relationships as edges with edge types and properties directly attached to them.\nAdvantages of NebulaGraph\nOpen source\nNebulaGraph is open under the Apache 2.0 License. More and more people such as database developers, data scientists, security experts, and algorithm engineers are participating in the designing and development of NebulaGraph. To join the opening of source code and ideas, surf the NebulaGraph GitHub page.\nOutstanding performance\nWritten in C++ and born for graphs, NebulaGraph handles graph queries in milliseconds. Among most databases, NebulaGraph shows superior performance in providing graph data services. The larger the data size, the greater the superiority of NebulaGraph.For more information, see NebulaGraph benchmarking.\nHigh scalability\nNebulaGraph is designed in a shared-nothing architecture and supports scaling in and out without interrupting the database service.\nDeveloper friendly\nNebulaGraph supports clients in popular programming languages like Java, Python, C++, and Go, and more are under development. For more information, see NebulaGraph clients.\nReliable access control\nNebulaGraph supports strict role-based access control and external authentication servers such as LDAP (Lightweight Directory Access Protocol) servers to enhance data security. For more information, see Authentication and authorization.\nDiversified ecosystem\nMore and more native tools of NebulaGraph have been released, such as NebulaGraph Studio, NebulaGraph Console, and NebulaGraph Exchange. For more ecosystem tools, see Ecosystem tools overview.\nBesides, NebulaGraph has the ability to be integrated with many cutting-edge technologies, such as Spark, Flink, and HBase, for the purpose of mutual strengthening in a world of increasing challenges and chances.\nOpenCypher-compatible query language\nThe native NebulaGraph Query Language, also known as nGQL, is a declarative, openCypher-compatible textual query language. It is easy to understand and easy to use. For more information, see nGQL guide.\nFuture-oriented hardware with balanced reading and writing\nSolid-state drives have extremely high performance and they are getting cheaper. NebulaGraph is a product based on SSD. Compared with products based on HDD and large memory, it is more suitable for future hardware trends and easier to achieve balanced reading and writing.\nEasy data modeling and high flexibility\nYou can easily model the connected data into NebulaGraph for your business without forcing them into a structure such as a relational table, and properties can be added, updated, and deleted freely. For more information, see Data modeling.\nHigh popularity\nNebulaGraph is being used by tech leaders such as Tencent, Vivo, Meituan, and JD Digits. For more information, visit the NebulaGraph official website.\nUse cases\nNebulaGraph can be used to support various graph-based scenarios. To spare the time spent on pushing the kinds of data mentioned in this section into relational databases and on bothering with join queries, use NebulaGraph.\nFraud detection\nFinancial institutions have to traverse countless transactions to piece together potential crimes and understand how combinations of transactions and devices might be related to a single fraud scheme. This kind of scenario can be modeled in graphs, and with the help of NebulaGraph, fraud rings and other sophisticated scams can be easily detected.\nReal-time recommendation\nNebulaGraph offers the ability to instantly process the real-time information produced by a visitor and make accurate recommendations on articles, videos, products, and services.\nIntelligent question-answer system\nNatural languages can be transformed into knowledge graphs and stored in NebulaGraph. A question organized in a natural language can be resolved by a semantic parser in an intelligent question-answer system and re-organized. Then, possible answers to the question can be retrieved from the knowledge graph and provided to the one who asked the question.\nSocial networking\nInformation on people and their relationships is typical graph data. NebulaGraph can easily handle the social networking information of billions of people and trillions of relationships, and provide lightning-fast queries for friend recommendations and job promotions in the case of massive concurrency.\nRelated links\nOfficial website\nDocs\nBlogs\nForum\nGitHub","url":"https://docs.nebula-graph.io/3.6.0/1.introduction/1.what-is-nebula-graph/","type":"doc"},{"title":"Data modeling","content":"A data model is a model that organizes data and specifies how they are related to one another. This topic describes the Nebula Graph data model and provides suggestions for data modeling with NebulaGraph.\nData structures\nNebulaGraph data model uses six data structures to store data. They are graph spaces, vertices, edges, tags, edge types and properties.\nGraph spaces: Graph spaces are used to isolate data from different teams or programs. Data stored in different graph spaces are securely isolated. Storage replications, privileges, and partitions can be assigned.\nVertices: Vertices are used to store entities.\nIn NebulaGraph, vertices are identified with vertex identifiers (i.e. VID). The VID must be unique in the same graph space. VID should be int64, or fixed_string(N).\nA vertex has zero to multiple tags.\nEdges: Edges are used to connect vertices. An edge is a connection or behavior between two vertices.\nThere can be multiple edges between two vertices.\nEdges are directed. -> identifies the directions of edges. Edges can be traversed in either direction.\nAn edge is identified uniquely with . Edges have no EID.\nAn edge must have one and only one edge type.\nThe rank value is an immutable user-assigned 64-bit signed integer. It identifies the edges with the same edge type between two vertices. Edges are sorted by their rank values. The edge with the greatest rank value is listed first. The default rank value is zero.\nTags: Tags are used to categorize vertices. Vertices that have the same tag share the same definition of properties.\nEdge types: Edge types are used to categorize edges. Edges that have the same edge type share the same definition of properties.\nProperties: Properties are key-value pairs. Both vertices and edges are containers for properties.\nDirected property graph\nNebulaGraph stores data in directed property graphs. A directed property graph has a set of vertices connected by directed edges. Both vertices and edges can have properties. A directed property graph is represented as:\nG = < V, E, PV, PE >\nV is a set of vertices.\nE is a set of directed edges.\nPV is the property of vertices.\nPE is the property of edges.\nThe following table is an example of the structure of the basketball player dataset. We have two types of vertices, that is player and team, and two types of edges, that is serve and follow.\nElement\nName\nProperty name (Data type)\nDescription\nTag\nplayer\nname (string) age (int)\nRepresents players in the team.\nTag\nteam\nname (string)\nRepresents the teams.\nEdge type\nserve\nstart_year (int) end_year (int)\nRepresents actions taken by players in the team.An action links a player with a team, and the direction is from a player to a team.\nEdge type\nfollow\ndegree (int)\nRepresents actions taken by players in the team.An action links a player with another player, and the direction is from one player to the other player.","url":"https://docs.nebula-graph.io/3.6.0/1.introduction/2.data-model/","type":"doc"},{"title":"Path types","content":"In graph theory, a path in a graph is a finite or infinite sequence of edges which joins a sequence of vertices. Paths are fundamental concepts of graph theory.\nPaths can be categorized into 3 types: walk, trail, and path. For more information, see Wikipedia.\nThe following figure is an example for a brief introduction.\nWalk\nA walk is a finite or infinite sequence of edges. Both vertices and edges can be repeatedly visited in graph traversal.\nIn the above figure C, D, and E form a cycle. So, this figure contains infinite paths, such as A->B->C->D->E, A->B->C->D->E->C, and A->B->C->D->E->C->D.\nTrail\nA trail is a finite sequence of edges. Only vertices can be repeatedly visited in graph traversal. The Seven Bridges of Königsberg is a typical trail.\nIn the above figure, edges cannot be repeatedly visited. So, this figure contains finite paths. The longest path in this figure consists of 5 edges: A->B->C->D->E->C.\nThere are two special cases of trail, cycle and circuit. The following figure is an example for a brief introduction.\ncycle\nA cycle refers to a closed trail. Only the terminal vertices can be repeatedly visited. The longest path in this figure consists of 3 edges: A->B->C->A or C->D->E->C.\ncircuit\nA circuit refers to a closed trail. Edges cannot be repeatedly visited in graph traversal. Apart from the terminal vertices, other vertices can also be repeatedly visited. The longest path in this figure: A->B->C->D->E->C->A.\nPath\nA path is a finite sequence of edges. Neither vertices nor edges can be repeatedly visited in graph traversal.\nSo, the above figure contains finite paths. The longest path in this figure consists of 4 edges: A->B->C->D->E.","url":"https://docs.nebula-graph.io/3.6.0/1.introduction/2.1.path/","type":"doc"},{"title":"VID","content":"In a graph space, a vertex is uniquely identified by its ID, which is called a VID or a Vertex ID.\nFeatures\nThe data types of VIDs are restricted to FIXED_STRING() or INT64. One graph space can only select one VID type.\nA VID in a graph space is unique. It functions just as a primary key in a relational database. VIDs in different graph spaces are independent.\nThe VID generation method must be set by users, because NebulaGraph does not provide auto increasing ID, or UUID.\nVertices with the same VID will be identified as the same one. For example:\nA VID is the unique identifier of an entity, like a person's ID card number. A tag means the type of an entity, such as driver, and boss. Different tags define two groups of different properties, such as driving license number, driving age, order amount, order taking alt, and job number, payroll, debt ceiling, business phone number.\nWhen two INSERT statements (neither uses a parameter of IF NOT EXISTS) with the same VID and tag are operated at the same time, the latter INSERT will overwrite the former.\nWhen two INSERT statements with the same VID but different tags, like TAG A and TAG B, are operated at the same time, the operation of Tag A will not affect Tag B.\nVIDs will usually be indexed and stored into memory (in the way of LSM-tree). Thus, direct access to VIDs enjoys peak performance.\nVID Operation\nNebulaGraph 1.x only supports INT64 while NebulaGraph 2.x supports INT64 and FIXED_STRING(). In CREATE SPACE, VID types can be set via vid_type.\nid() function can be used to specify or locate a VID.\nLOOKUP or MATCH statements can be used to find a VID via property index.\nDirect access to vertices statements via VIDs enjoys peak performance, such as DELETE xxx WHERE id(xxx) == \"player100\" or GO FROM \"player100\". Finding VIDs via properties and then operating the graph will cause poor performance, such as LOOKUP | GO FROM $-.ids, which will run both LOOKUP and | one more time.\nVID Generation\nVIDs can be generated via applications. Here are some tips:\n(Optimal) Directly take a unique primary key or property as a VID. Property access depends on the VID.\nGenerate a VID via a unique combination of properties. Property access depends on property index.\nGenerate a VID via algorithms like snowflake. Property access depends on property index.\nIf short primary keys greatly outnumber long primary keys, do not enlarge the N of FIXED_STRING() too much. Otherwise, it will occupy a lot of memory and hard disks, and slow down performance. Generate VIDs via BASE64, MD5, hash by encoding and splicing.\nIf you generate int64 VID via hash, the probability of collision is about 1/10 when there are 1 billion vertices. The number of edges has no concern with the probability of collision.\nDefine and modify a VID and its data type\nThe data type of a VID must be defined when you create the graph space. Once defined, it cannot be modified.\nA VID is set when you insert a vertex and cannot be modified. \nQuery start vid and global scan\nIn most cases, the execution plan of query statements in NebulaGraph (MATCH, GO, and LOOKUP) must query the start vid in a certain way.\nThere are only two ways to locate start vid:\nFor example, GO FROM \"player100\" OVER explicitly indicates in the statement that start vid is \"player100\".\nFor example, LOOKUP ON player WHERE player.name == \"Tony Parker\" or MATCH (v:player {name:\"Tony Parker\"}) locates start vid by the index of the property player.name.","url":"https://docs.nebula-graph.io/3.6.0/1.introduction/3.vid/","type":"doc"},{"title":"Architecture overview","content":"NebulaGraph consists of three services: the Graph Service, the Storage Service, and the Meta Service. It applies the separation of storage and computing architecture.\nEach service has its executable binaries and processes launched from the binaries. Users can deploy a NebulaGraph cluster on a single machine or multiple machines using these binaries.\nThe following figure shows the architecture of a typical NebulaGraph cluster.\nThe Meta Service\nThe Meta Service in the NebulaGraph architecture is run by the nebula-metad processes. It is responsible for metadata management, such as schema operations, cluster administration, and user privilege management.\nFor details on the Meta Service, see Meta Service.\nThe Graph Service and the Storage Service\nNebulaGraph applies the separation of storage and computing architecture. The Graph Service is responsible for querying. The Storage Service is responsible for storage. They are run by different processes, i.e., nebula-graphd and nebula-storaged. The benefits of the separation of storage and computing architecture are as follows:\nGreat scalabilityThe separated structure makes both the Graph Service and the Storage Service flexible and easy to scale in or out.\nHigh availabilityIf part of the Graph Service fails, the data stored by the Storage Service suffers no loss. And if the rest part of the Graph Service is still able to serve the clients, service recovery can be performed quickly, even unfelt by the users.\nCost-effectiveThe separation of storage and computing architecture provides a higher resource utilization rate, and it enables clients to manage the cost flexibly according to business demands.\nOpen to more possibilitiesWith the ability to run separately, the Graph Service may work with multiple types of storage engines, and the Storage Service may also serve more types of computing engines.\nFor details on the Graph Service and the Storage Service, see Graph Service and Storage Service.","url":"https://docs.nebula-graph.io/3.6.0/1.introduction/3.nebula-graph-architecture/1.architecture-overview/","type":"doc"},{"title":"Meta Service","content":"This topic introduces the architecture and functions of the Meta Service.\nThe architecture of the Meta Service\nThe architecture of the Meta Service is as follows:\nThe Meta Service is run by nebula-metad processes. Users can deploy nebula-metad processes according to the scenario:\nIn a test environment, users can deploy one or three nebula-metad processes on different machines or a single machine.\nIn a production environment, we recommend that users deploy three nebula-metad processes on different machines for high availability.\nAll the nebula-metad processes form a Raft-based cluster, with one process as the leader and the others as the followers.\nThe leader is elected by the majorities and only the leader can provide service to the clients or other components of NebulaGraph. The followers will be run in a standby way and each has a data replication of the leader. Once the leader fails, one of the followers will be elected as the new leader.\nFunctions of the Meta Service\nManages user accounts\nThe Meta Service stores the information of user accounts and the privileges granted to the accounts. When the clients send queries to the Meta Service through an account, the Meta Service checks the account information and whether the account has the right privileges to execute the queries or not.\nFor more information on NebulaGraph access control, see Authentication.\nManages partitions\nThe Meta Service stores and manages the locations of the storage partitions and helps balance the partitions.\nManages graph spaces\nNebulaGraph supports multiple graph spaces. Data stored in different graph spaces are securely isolated. The Meta Service stores the metadata of all graph spaces and tracks the changes of them, such as adding or dropping a graph space.\nManages schema information\nNebulaGraph is a strong-typed graph database. Its schema contains tags (i.e., the vertex types), edge types, tag properties, and edge type properties.\nThe Meta Service stores the schema information. Besides, it performs the addition, modification, and deletion of the schema, and logs the versions of them.\nFor more information on NebulaGraph schema, see Data model.\nManages TTL information\nThe Meta Service stores the definition of TTL (Time to Live) options which are used to control data expiration. The Storage Service takes care of the expiring and evicting processes. For more information, see TTL.\nManages jobs\nThe Job Management module in the Meta Service is responsible for the creation, queuing, querying, and deletion of jobs.","url":"https://docs.nebula-graph.io/3.6.0/1.introduction/3.nebula-graph-architecture/2.meta-service/","type":"doc"},{"title":"Graph Service","content":"The Graph Service is used to process the query. It has four submodules: Parser, Validator, Planner, and Executor. This topic will describe the Graph Service accordingly.\nThe architecture of the Graph Service\nAfter a query is sent to the Graph Service, it will be processed by the following four submodules:\nParser: Performs lexical analysis and syntax analysis.\nValidator: Validates the statements.\nPlanner: Generates and optimizes the execution plans.\nExecutor: Executes the plans with operators.\nParser\nAfter receiving a request, the statements will be parsed by Parser composed of Flex (lexical analysis tool) and Bison (syntax analysis tool), and its corresponding AST will be generated. Statements will be directly intercepted in this stage because of their invalid syntax.\nFor example, the structure of the AST of GO FROM \"Tim\" OVER like WHERE properties(edge).likeness > 8.0 YIELD dst(edge) is shown in the following figure.\nValidator\nValidator performs a series of validations on the AST. It mainly works on these tasks:\nValidating metadataValidator will validate whether the metadata is correct or not.\nWhen parsing the OVER, WHERE, and YIELD clauses, Validator looks up the Schema and verifies whether the edge type and tag data exist or not. For an INSERT statement, Validator verifies whether the types of the inserted data are the same as the ones defined in the Schema.\nValidating contextual referenceValidator will verify whether the cited variable exists or not, or whether the cited property is variable or not.\nFor composite statements, like $var = GO FROM \"Tim\" OVER like YIELD dst(edge) AS ID; GO FROM $var.ID OVER serve YIELD dst(edge), Validator verifies first to see if var is defined, and then to check if the ID property is attached to the var variable.\nValidating type inferenceValidator infers what type the result of an expression is and verifies the type against the specified clause.\nFor example, the WHERE clause requires the result to be a bool value, a NULL value, or empty.\nValidating the information of *Validator needs to verify all the Schema that involves * when verifying the clause if there is a * in the statement.\nTake a statement like GO FROM \"Tim\" OVER * YIELD dst(edge), properties(edge).likeness, dst(edge) as an example. When verifying the OVER clause, Validator needs to verify all the edge types. If the edge type includes like and serve, the statement would be GO FROM \"Tim\" OVER like,serve YIELD dst(edge), properties(edge).likeness, dst(edge).\nValidating input and outputValidator will check the consistency of the clauses before and after the |.\nIn the statement GO FROM \"Tim\" OVER like YIELD dst(edge) AS ID | GO FROM $-.ID OVER serve YIELD dst(edge), Validator will verify whether $-.ID is defined in the clause before the |.\nWhen the validation succeeds, an execution plan will be generated. Its data structure will be stored in the src/planner directory.\nPlanner\nIn the nebula-graphd.conf file, when enable_optimizer is set to be false, Planner will not optimize the execution plans generated by Validator. It will be executed by Executor directly.\nIn the nebula-graphd.conf file, when enable_optimizer is set to be true, Planner will optimize the execution plans generated by Validator. The structure is as follows.\nBefore optimizationIn the execution plan on the right side of the preceding figure, each node directly depends on other nodes. For example, the root node Project depends on the Filter node, the Filter node depends on the GetNeighbor node, and so on, up to the leaf node Start. Then the execution plan is (not truly) executed.\nDuring this stage, every node has its input and output variables, which are stored in a hash table. The execution plan is not truly executed, so the value of each key in the associated hash table is empty (except for the Start node, where the input variables hold the starting data), and the hash table is defined in src/context/ExecutionContext.cpp under the nebula-graph repository.\nFor example, if the hash table is named as ResultMap when creating the Filter node, users can determine that the node takes data from ResultMap[\"GN1\"], then puts the result into ResultMap[\"Filter2\"], and so on. All these work as the input and output of each node.\nProcess of optimizationThe optimization rules that Planner has implemented so far are considered RBO (Rule-Based Optimization), namely the pre-defined optimization rules. The CBO (Cost-Based Optimization) feature is under development. The optimized code is in the src/optimizer/ directory under the nebula-graph repository.\nRBO is a “bottom-up” exploration process. For each rule, the root node of the execution plan (in this case, the Project node) is the entry point, and step by step along with the node dependencies, it reaches the node at the bottom to see if it matches the rule.\nAs shown in the preceding figure, when the Filter node is explored, it is found that its children node is GetNeighbors, which matches successfully with the pre-defined rules, so a transformation is initiated to integrate the Filter node into the GetNeighbors node, the Filter node is removed, and then the process continues to the next rule. Therefore, when the GetNeighbor operator calls interfaces of the Storage layer to get the neighboring edges of a vertex during the execution stage, the Storage layer will directly filter out the unqualified edges internally. Such optimization greatly reduces the amount of data transfer, which is commonly known as filter pushdown.\nExecutor\nThe Executor module consists of Scheduler and Executor. The Scheduler generates the corresponding execution operators against the execution plan, starting from the leaf nodes and ending at the root node. The structure is as follows.\nEach node of the execution plan has one execution operator node, whose input and output have been determined in the execution plan. Each operator only needs to get the values for the input variables, compute them, and finally put the results into the corresponding output variables. Therefore, it is only necessary to execute step by step from Start, and the result of the last operator is returned to the user as the final result.\nSource code hierarchy\nThe source code hierarchy under the nebula-graph repository is as follows.\n|--src\n |--graph\n |--context //contexts for validation and execution\n |--executor //execution operators\n |--gc //garbage collector\n |--optimizer //optimization rules\n |--planner //structure of the execution plans\n |--scheduler //scheduler\n |--service //external service management\n |--session //session management\n |--stats //monitoring metrics\n |--util //basic components\n |--validator //validation of the statements\n |--visitor //visitor expression","url":"https://docs.nebula-graph.io/3.6.0/1.introduction/3.nebula-graph-architecture/3.graph-service/","type":"doc"},{"title":"Storage Service","content":"The persistent data of NebulaGraph have two parts. One is the Meta Service that stores the meta-related data.\nThe other is the Storage Service that stores the data, which is run by the nebula-storaged process. This topic will describe the architecture of the Storage Service.\nAdvantages\nHigh performance (Customized built-in KVStore)\nGreat scalability (Shared-nothing architecture, not rely on NAS/SAN-like devices)\nStrong consistency (Raft)\nHigh availability (Raft)\nSupports synchronizing with the third party systems, such as Elasticsearch.\nThe architecture of the Storage Service\nThe Storage Service is run by the nebula-storaged process. Users can deploy nebula-storaged processes on different occasions. For example, users can deploy 1 nebula-storaged process in a test environment and deploy 3 nebula-storaged processes in a production environment.\nAll the nebula-storaged processes consist of a Raft-based cluster. There are three layers in the Storage Service:\nStorage interface\nThe top layer is the storage interface. It defines a set of APIs that are related to the graph concepts. These API requests will be translated into a set of KV operations targeting the corresponding Partition. For example:\ngetNeighbors: queries the in-edge or out-edge of a set of vertices, returns the edges and the corresponding properties, and supports conditional filtering.\ninsert vertex/edge: inserts a vertex or edge and its properties.\ngetProps: gets the properties of a vertex or an edge.\nIt is this layer that makes the Storage Service a real graph storage. Otherwise, it is just a KV storage.\nConsensus\nBelow the storage interface is the consensus layer that implements Multi Group Raft, which ensures the strong consistency and high availability of the Storage Service.\nStore engine\nThe bottom layer is the local storage engine library, providing operations like get, put, and scan on local disks. The related interfaces are stored in KVStore.h and KVEngine.h files. You can develop your own local store plugins based on your needs.\nThe following will describe some features of the Storage Service based on the above architecture.\nStorage writing process\nKVStore\nNebulaGraph develops and customizes its built-in KVStore for the following reasons.\nIt is a high-performance KVStore.\nIt is provided as a (kv) library and can be easily developed for the filter pushdown purpose. As a strong-typed database, how to provide Schema during pushdown is the key to efficiency for NebulaGraph.\nIt has strong data consistency.\nTherefore, NebulaGraph develops its own KVStore with RocksDB as the local storage engine. The advantages are as follows.\nFor multiple local hard disks, NebulaGraph can make full use of its concurrent capacities through deploying multiple data directories.\nThe Meta Service manages all the Storage servers. All the partition distribution data and current machine status can be found in the meta service. Accordingly, users can execute a manual load balancing plan in meta service.\nNebulaGraph provides its own WAL mode so one can customize the WAL. Each partition owns its WAL.\nOne NebulaGraph KVStore cluster supports multiple graph spaces, and each graph space has its own partition number and replica copies. Different graph spaces are isolated physically from each other in the same cluster.\nData storage structure\nGraphs consist of vertices and edges. NebulaGraph uses key-value pairs to store vertices, edges, and their properties. Vertices and edges are stored in keys and their properties are stored in values. Such structure enables efficient property filtering.\nThe storage structure of vertices\nDifferent from NebulaGraph version 2.x, version 3.x added a new key for each vertex. Compared to the old key that still exists, the new key has no TagID field and no value. Vertices in NebulaGraph can now live without tags owing to the new key.\nField\nDescription\nType\nOne byte, used to indicate the key type.\nPartID\nThree bytes, used to indicate the sharding partition and to scan the partition data based on the prefix when re-balancing the partition.\nVertexID\nThe vertex ID. For an integer VertexID, it occupies eight bytes. However, for a string VertexID, it is changed to fixed_string of a fixed length which needs to be specified by users when they create the space.\nTagID\nFour bytes, used to indicate the tags that vertex relate with.\nSerializedValue\nThe serialized value of the key. It stores the property information of the vertex.\nThe storage structure of edges\nField\nDescription\nType\nOne byte, used to indicate the key type.\nPartID\nThree bytes, used to indicate the partition ID. This field can be used to scan the partition data based on the prefix when re-balancing the partition.\nVertexID\nUsed to indicate vertex ID. The former VID refers to the source VID in the outgoing edge and the dest VID in the incoming edge, while the latter VID refers to the dest VID in the outgoing edge and the source VID in the incoming edge.\nEdge Type\nFour bytes, used to indicate the edge type. Greater than zero indicates out-edge, less than zero means in-edge.\nRank\nEight bytes, used to indicate multiple edges in one edge type. Users can set the field based on needs and store weight, such as transaction time and transaction number.\nPlaceHolder\nOne byte. Reserved.\nSerializedValue\nThe serialized value of the key. It stores the property information of the edge.\nProperty descriptions\nNebulaGraph uses strong-typed Schema.\nNebulaGraph will store the properties of vertex and edges in order after encoding them. Since the length of fixed-length properties is fixed, queries can be made in no time according to offset. Before decoding, NebulaGraph needs to get (and cache) the schema information in the Meta Service. In addition, when encoding properties, NebulaGraph will add the corresponding schema version to support online schema change.\nData partitioning\nSince in an ultra-large-scale relational network, vertices can be as many as tens to hundreds of billions, and edges are even more than trillions. Even if only vertices and edges are stored, the storage capacity of both exceeds that of ordinary servers. Therefore, NebulaGraph uses hash to shard the graph elements and store them in different partitions.\nEdge partitioning and storage amplification\nIn NebulaGraph, an edge corresponds to two key-value pairs on the hard disk. When there are lots of edges and each has many properties, storage amplification will be obvious. The storage format of edges is shown in the figure below.\nIn this example, SrcVertex connects DstVertex via EdgeA, forming the path of (SrcVertex)-[EdgeA]->(DstVertex). SrcVertex, DstVertex, and EdgeA will all be stored in Partition x and Partition y as four key-value pairs in the storage layer. Details are as follows:\nThe key value of SrcVertex is stored in Partition x. Key fields include Type, PartID(x), VID(Src), and TagID. SerializedValue, namely Value, refers to serialized vertex properties.\nThe first key value of EdgeA, namely EdgeA_Out, is stored in the same partition as the SrcVertex. Key fields include Type, PartID(x), VID(Src), EdgeType(+ means out-edge), Rank(0), VID(Dst), and PlaceHolder. SerializedValue, namely Value, refers to serialized edge properties.\nThe key value of DstVertex is stored in Partition y. Key fields include Type, PartID(y), VID(Dst), and TagID. SerializedValue, namely Value, refers to serialized vertex properties.\nThe second key value of EdgeA, namely EdgeA_In, is stored in the same partition as the DstVertex. Key fields include Type, PartID(y), VID(Dst), EdgeType(- means in-edge), Rank(0), VID(Src), and PlaceHolder. SerializedValue, namely Value, refers to serialized edge properties, which is exactly the same as that in EdgeA_Out.\nEdgeA_Out and EdgeA_In are stored in storage layer with opposite directions, constituting EdgeA logically. EdgeA_Out is used for traversal requests starting from SrcVertex, such as (a)-[]->(); EdgeA_In is used for traversal requests starting from DstVertex, such as ()-[]->(a).\nLike EdgeA_Out and EdgeA_In, NebulaGraph redundantly stores the information of each edge, which doubles the actual capacities needed for edge storage. The key corresponding to the edge occupies a small hard disk space, but the space occupied by Value is proportional to the length and amount of the property value. Therefore, it will occupy a relatively large hard disk space if the property value of the edge is large or there are many edge property values.\nPartition algorithm\nNebulaGraph uses a static Hash strategy to shard data through a modulo operation on vertex ID. All the out-keys, in-keys, and tag data will be placed in the same partition. In this way, query efficiency is increased dramatically.\nWhen inserting into NebulaGraph, vertices and edges are distributed across different partitions. And the partitions are located on different machines. The number of partitions is set in the CREATE SPACE statement and cannot be changed afterward.\nIf certain vertices need to be placed on the same partition (i.e., on the same machine), see Formula/code.\nThe following code will briefly describe the relationship between VID and partition.\n// If VertexID occupies 8 bytes, it will be stored in int64 to be compatible with the version 1.0.\nuint64_t vid = 0;\nif (id.size() == 8) {\n memcpy(static_cast(&vid), id.data(), 8);\n} else {\n MurmurHash2 hash;\n vid = hash(id.data());\nPartitionID pId = vid % numParts + 1;\nRoughly speaking, after hashing a fixed string to int64, (the hashing of int64 is the number itself), do modulo, and then plus one, namely:\npId = vid % numParts + 1;\nParameters and descriptions of the preceding formula are as follows:\nParameter\nDescription\nThe modulo operation.\nnumParts\nThe number of partitions for the graph space where the VID is located, namely the value of partition_num in the CREATE SPACE statement.\npId\nThe ID for the partition where the VID is located.\nSuppose there are 100 partitions, the vertices with VID 1, 101, and 1001 will be stored on the same partition. But, the mapping between the partition ID and the machine address is random. Therefore, we cannot assume that any two partitions are located on the same machine.\nRaft\nRaft implementation\nIn a distributed system, one data usually has multiple replicas so that the system can still run normally even if a few copies fail. It requires certain technical means to ensure consistency between replicas.\nBasic principle: Raft is designed to ensure consistency between replicas. Raft uses election between replicas, and the (candidate) replica that wins more than half of the votes will become the Leader, providing external services on behalf of all replicas. The rest Followers will play backups. When the Leader fails (due to communication failure, operation and maintenance commands, etc.), the rest Followers will conduct a new round of elections and vote for a new Leader. The Leader and Followers will detect each other's survival through heartbeats and write them to the hard disk in Raft-wal mode. Replicas that do not respond to more than multiple heartbeats will be considered faulty.\nRead and write: For every writing request of the clients, the Leader will initiate a Raft-wal and synchronize it with the Followers. Only after over half replicas have received the Raft-wal will it return to the clients successfully. For every reading request of the clients, it will get to the Leader directly, while Followers will not be involved.\nFailure: Scenario 1: Take a (space) cluster of a single replica as an example. If the system has only one replica, the Leader will be itself. If failure happens, the system will be completely unavailable. Scenario 2: Take a (space) cluster of three replicas as an example. If the system has three replicas, one of them will be the Leader and the rest will be the Followers. If the Leader fails, the rest two can still vote for a new Leader (and a Follower), and the system is still available. But if any of the two Followers fails again, the system will be completely unavailable due to inadequate voters.\nMulti Group Raft\nThe Storage Service supports a distributed cluster architecture, so NebulaGraph implements Multi Group Raft according to Raft protocol. Each Raft group stores all the replicas of each partition. One replica is the leader, while others are followers. In this way, NebulaGraph achieves strong consistency and high availability. The functions of Raft are as follows.\nNebulaGraph uses Multi Group Raft to improve performance when there are many partitions because Raft-wal cannot be NULL. When there are too many partitions, costs will increase, such as storing information in Raft group, WAL files, or batch operation in low load.\nThere are two key points to implement the Multi Raft Group:\nTo share transport layer\nEach Raft Group sends messages to its corresponding peers. So if the transport layer cannot be shared, the connection costs will be very high.\nTo share thread pool\nRaft Groups share the same thread pool to prevent starting too many threads and a high context switch cost.\nBatch\nFor each partition, it is necessary to do a batch to improve throughput when writing the WAL serially. As NebulaGraph uses WAL to implement some special functions, batches need to be grouped, which is a feature of NebulaGraph.\nFor example, lock-free CAS operations will execute after all the previous WALs are committed. So for a batch, if there are several WALs in CAS type, we need to divide this batch into several smaller groups and make sure they are committed serially.\nTransfer Leadership\nTransfer leadership is extremely important for balance. When moving a partition from one machine to another, NebulaGraph first checks if the source is a leader. If so, it should be moved to another peer. After data migration is completed, it is important to balance leader distribution again.\nWhen a transfer leadership command is committed, the leader will abandon its leadership and the followers will start a leader election.\nPeer changes\nTo avoid split-brain, when members in a Raft Group change, an intermediate state is required. In such a state, the quorum of the old group and new group always have an overlap. Thus it prevents the old or new group from making decisions unilaterally. To make it even simpler, in his doctoral thesis Diego Ongaro suggests adding or removing a peer once to ensure the overlap between the quorum of the new group and the old group. NebulaGraph also uses this approach, except that the way to add or remove a member is different. For details, please refer to addPeer/removePeer in the Raft Part class.\nDifferences with HDFS\nThe Storage Service is a Raft-based distributed architecture, which has certain differences with that of HDFS. For example:\nThe Storage Service ensures consistency through Raft. Usually, the number of its replicas is odd to elect a leader. However, DataNode used by HDFS ensures consistency through NameNode, which has no limit on the number of replicas.\nIn the Storage Service, only the replicas of the leader can read and write, while in HDFS all the replicas can do so.\nIn the Storage Service, the number of replicas needs to be determined when creating a space, since it cannot be changed afterward. But in HDFS, the number of replicas can be changed freely.\nThe Storage Service can access the file system directly. While the applications of HDFS (such as HBase) have to access HDFS before the file system, which requires more RPC times.\nIn a word, the Storage Service is more lightweight with some functions simplified and its architecture is simpler than HDFS, which can effectively improve the read and write performance of a smaller block of data.","url":"https://docs.nebula-graph.io/3.6.0/1.introduction/3.nebula-graph-architecture/4.storage-service/","type":"doc"},{"title":"Quickly deploy NebulaGraph using Docker","content":"You can quickly get started with NebulaGraph by deploying NebulaGraph with Docker Desktop or Docker Compose. \nUsing Docker DesktopUsing Docker Compose\nNebulaGraph is available as a Docker Extension that you can easily install and run on your Docker Desktop. You can quickly deploy NebulaGraph using Docker Desktop with just one click.\nInstall Docker Desktop\nInstall Docker Desktop on Mac\nInstall Docker Desktop on Windows\nIn the left sidebar of Docker Desktop, click Extensions or Add Extensions.\nOn the Extensions Marketplace, search for NebulaGraph and click Install.\nClick Update to update NebulaGraph to the latest version when a new version is available.\nClick Open to navigate to the NebulaGraph extension page.\nAt the top of the page, click Studio in Browser to use NebulaGraph.\nFor more information about how to use NebulaGraph with Docker Desktop, see the following video:\nUsing Docker Compose can quickly deploy NebulaGraph services based on the prepared configuration file. It is only recommended to use this method when testing the functions of NebulaGraph.\nPrerequisites\nYou have installed the following applications on your host.\nApplication\nRecommended version\nOfficial installation reference\nDocker\nLatest\nInstall Docker Engine\nDocker Compose\nLatest\nInstall Docker Compose\nGit\nLatest\nDownload Git\nIf you are deploying NebulaGraph as a non-root user, grant the user with Docker-related privileges. For detailed instructions, see Manage Docker as a non-root user.\nYou have started the Docker service on your host.\nIf you have already deployed another version of NebulaGraph with Docker Compose on your host, to avoid compatibility issues, you need to delete the nebula-docker-compose/data directory.\nDeploy NebulaGraph\nClone the 3.6.0 branch of the nebula-docker-compose repository to your host with Git.\n$ git clone -b release-3.6 https://github.com/vesoft-inc/nebula-docker-compose.git\nGo to the nebula-docker-compose directory.\n$ cd nebula-docker-compose/\nRun the following command to start all the NebulaGraph services.\n[nebula-docker-compose]$ docker-compose up -d\nCreating nebula-docker-compose_metad0_1 ... done\nCreating nebula-docker-compose_metad2_1 ... done\nCreating nebula-docker-compose_metad1_1 ... done\nCreating nebula-docker-compose_graphd2_1 ... done\nCreating nebula-docker-compose_graphd_1 ... done\nCreating nebula-docker-compose_graphd1_1 ... done\nCreating nebula-docker-compose_storaged0_1 ... done\nCreating nebula-docker-compose_storaged2_1 ... done\nCreating nebula-docker-compose_storaged1_1 ... done\nConnect to NebulaGraph\nThere are two ways to connect to NebulaGraph:\nConnected with Nebula Console outside the container. Because the external mapping port for the Graph service is also fixed as 9669 in the container's configuration file, you can connect directly through the default port. For details, see Connect to NebulaGraph.\nLog into the container installed NebulaGraph Console, then connect to the Graph service. This section describes this approach.\nRun the following command to view the name of NebulaGraph Console docker container.\n$ docker-compose ps\n Name Command State Ports\n--------------------------------------------------------------------------------------------\nnebula-docker-compose_console_1 sh -c sleep 3 && Up\n nebula-co ...\n......\nRun the following command to enter the NebulaGraph Console docker container.\ndocker exec -it nebula-docker-compose_console_1 /bin/sh\n/ #\nConnect to NebulaGraph with NebulaGraph Console.\n/ # ./usr/local/bin/nebula-console -u -p --address=graphd --port=9669\nRun the following commands to view the cluster state.\n SHOW HOSTS;\nRun exit twice to switch back to your terminal (shell).\nCheck the NebulaGraph service status and ports\nRun docker-compose ps to list all the services of NebulaGraph and their status and ports.\n$ docker-compose ps\nnebula-docker-compose_console_1 sh -c sleep 3 && Up\n nebula-co ...\nnebula-docker-compose_graphd1_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49174->19669/tcp,:::49174->19669/tcp, 0.0.0.0:49171->19670/tcp,:::49171->19670/tcp, 0.0.0.0:49177->9669/tcp,:::49177->9669/tcp\nnebula-docker-compose_graphd2_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49175->19669/tcp,:::49175->19669/tcp, 0.0.0.0:49172->19670/tcp,:::49172->19670/tcp, 0.0.0.0:49178->9669/tcp,:::49178->9669/tcp\nnebula-docker-compose_graphd_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49180->19669/tcp,:::49180->19669/tcp, 0.0.0.0:49179->19670/tcp,:::49179->19670/tcp, 0.0.0.0:9669->9669/tcp,:::9669->9669/tcp\nnebula-docker-compose_metad0_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49157->19559/tcp,:::49157->19559/tcp, 0.0.0.0:49154->19560/tcp,:::49154->19560/tcp, 0.0.0.0:49160->9559/tcp,:::49160->9559/tcp, 9560/tcp\nnebula-docker-compose_metad1_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49156->19559/tcp,:::49156->19559/tcp, 0.0.0.0:49153->19560/tcp,:::49153->19560/tcp, 0.0.0.0:49159->9559/tcp,:::49159->9559/tcp, 9560/tcp\nnebula-docker-compose_metad2_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49158->19559/tcp,:::49158->19559/tcp, 0.0.0.0:49155->19560/tcp,:::49155->19560/tcp, 0.0.0.0:49161->9559/tcp,:::49161->9559/tcp, 9560/tcp\nnebula-docker-compose_storaged0_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49166->19779/tcp,:::49166->19779/tcp, 0.0.0.0:49163->19780/tcp,:::49163->19780/tcp, 9777/tcp, 9778/tcp, 0.0.0.0:49169->9779/tcp,:::49169->9779/tcp, 9780/tcp\nnebula-docker-compose_storaged1_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49165->19779/tcp,:::49165->19779/tcp, 0.0.0.0:49162->19780/tcp,:::49162->19780/tcp, 9777/tcp, 9778/tcp, 0.0.0.0:49168->9779/tcp,:::49168->9779/tcp, 9780/tcp\nnebula-docker-compose_storaged2_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49167->19779/tcp,:::49167->19779/tcp, 0.0.0.0:49164->19780/tcp,:::49164->19780/tcp, 9777/tcp, 9778/tcp, 0.0.0.0:49170->9779/tcp,:::49170->9779/tcp, 9780/tcp\nIf the service is abnormal, you can first confirm the abnormal container name (such as nebula-docker-compose_graphd2_1).\nThen you can execute docker ps to view the corresponding CONTAINER ID (such as 2a6c56c405f5).\n[nebula-docker-compose]$ docker ps\nCONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\n2a6c56c405f5 vesoft/nebula-graphd:nightly \"/usr/local/nebula/b…\" 36 minutes ago Up 36 minutes (healthy) 0.0.0.0:49230->9669/tcp, 0.0.0.0:49229->19669/tcp, 0.0.0.0:49228->19670/tcp nebula-docker-compose_graphd2_1\n7042e0a8e83d vesoft/nebula-storaged:nightly \"./bin/nebula-storag…\" 36 minutes ago Up 36 minutes (healthy) 9777-9778/tcp, 9780/tcp, 0.0.0.0:49227->9779/tcp, 0.0.0.0:49226->19779/tcp, 0.0.0.0:49225->19780/tcp nebula-docker-compose_storaged2_1\n18e3ea63ad65 vesoft/nebula-storaged:nightly \"./bin/nebula-storag…\" 36 minutes ago Up 36 minutes (healthy) 9777-9778/tcp, 9780/tcp, 0.0.0.0:49219->9779/tcp, 0.0.0.0:49218->19779/tcp, 0.0.0.0:49217->19780/tcp nebula-docker-compose_storaged0_1\n4dcabfe8677a vesoft/nebula-graphd:nightly \"/usr/local/nebula/b…\" 36 minutes ago Up 36 minutes (healthy) 0.0.0.0:49224->9669/tcp, 0.0.0.0:49223->19669/tcp, 0.0.0.0:49222->19670/tcp nebula-docker-compose_graphd1_1\na74054c6ae25 vesoft/nebula-graphd:nightly \"/usr/local/nebula/b…\" 36 minutes ago Up 36 minutes (healthy) 0.0.0.0:9669->9669/tcp, 0.0.0.0:49221->19669/tcp, 0.0.0.0:49220->19670/tcp nebula-docker-compose_graphd_1\n880025a3858c vesoft/nebula-storaged:nightly \"./bin/nebula-storag…\" 36 minutes ago Up 36 minutes (healthy) 9777-9778/tcp, 9780/tcp, 0.0.0.0:49216->9779/tcp, 0.0.0.0:49215->19779/tcp, 0.0.0.0:49214->19780/tcp nebula-docker-compose_storaged1_1\n45736a32a23a vesoft/nebula-metad:nightly \"./bin/nebula-metad …\" 36 minutes ago Up 36 minutes (healthy) 9560/tcp, 0.0.0.0:49213->9559/tcp, 0.0.0.0:49212->19559/tcp, 0.0.0.0:49211->19560/tcp nebula-docker-compose_metad0_1\n3b2c90eb073e vesoft/nebula-metad:nightly \"./bin/nebula-metad …\" 36 minutes ago Up 36 minutes (healthy) 9560/tcp, 0.0.0.0:49207->9559/tcp, 0.0.0.0:49206->19559/tcp, 0.0.0.0:49205->19560/tcp nebula-docker-compose_metad2_1\n7bb31b7a5b3f vesoft/nebula-metad:nightly \"./bin/nebula-metad …\" 36 minutes ago Up 36 minutes (healthy) 9560/tcp, 0.0.0.0:49210->9559/tcp, 0.0.0.0:49209->19559/tcp, 0.0.0.0:49208->19560/tcp nebula-docker-compose_metad1_1\nUse the CONTAINER ID to log in the container and troubleshoot.\nnebula-docker-compose]$ docker exec -it 2a6c56c405f5 bash\n[root@2a6c56c405f5 nebula]#\nCheck the service data and logs\nAll the data and logs of NebulaGraph are stored persistently in the nebula-docker-compose/data and nebula-docker-compose/logs directories.\nThe structure of the directories is as follows:\nnebula-docker-compose/\n |-- docker-compose.yaml\n ├── data\n │   ├── meta0\n │   ├── meta1\n │   ├── meta2\n │   ├── storage0\n │   ├── storage1\n │   └── storage2\n └── logs\n ├── graph\n ├── graph1\n ├── graph2\n ├── meta0\n ├── meta1\n ├── meta2\n ├── storage0\n ├── storage1\n └── storage2\nStop the NebulaGraph services\nYou can run the following command to stop the NebulaGraph services:\n$ docker-compose down\nThe following information indicates you have successfully stopped the NebulaGraph services:\nStopping nebula-docker-compose_console_1 ... done\nStopping nebula-docker-compose_graphd1_1 ... done\nStopping nebula-docker-compose_graphd_1 ... done\nStopping nebula-docker-compose_graphd2_1 ... done\nStopping nebula-docker-compose_storaged1_1 ... done\nStopping nebula-docker-compose_storaged0_1 ... done\nStopping nebula-docker-compose_storaged2_1 ... done\nStopping nebula-docker-compose_metad2_1 ... done\nStopping nebula-docker-compose_metad0_1 ... done\nStopping nebula-docker-compose_metad1_1 ... done\nRemoving nebula-docker-compose_console_1 ... done\nRemoving nebula-docker-compose_graphd1_1 ... done\nRemoving nebula-docker-compose_graphd_1 ... done\nRemoving nebula-docker-compose_graphd2_1 ... done\nRemoving nebula-docker-compose_storaged1_1 ... done\nRemoving nebula-docker-compose_storaged0_1 ... done\nRemoving nebula-docker-compose_storaged2_1 ... done\nRemoving nebula-docker-compose_metad2_1 ... done\nRemoving nebula-docker-compose_metad0_1 ... done\nRemoving nebula-docker-compose_metad1_1 ... done\nRemoving network nebula-docker-compose_nebula-net\nModify configurations\nThe configuration file of NebulaGraph deployed by Docker Compose is nebula-docker-compose/docker-compose.yaml. To make the new configuration take effect, modify the configuration in this file and restart the service.\nFor more instructions, see Configurations.\nFAQ\nHow to fix the docker mapping to external ports?\nTo set the ports of corresponding services as fixed mapping, modify the docker-compose.yaml in the nebula-docker-compose directory. For example:\ngraphd:\n image: vesoft/nebula-graphd:release-3.6\n ...\n ports:\n - 9669:9669\n - 19669\n - 19670\n9669:9669 indicates the internal port 9669 is uniformly mapped to external ports, while 19669 indicates the internal port 19669 is randomly mapped to external ports.\nHow to upgrade or update the docker images of NebulaGraph services\nIn the nebula-docker-compose/docker-compose.yaml file, change all the image values to the required image version.\nIn the nebula-docker-compose directory, run docker-compose pull to update the images of the Graph Service, Storage Service, Meta Service, and NebulaGraph Console.\nRun docker-compose up -d to start the NebulaGraph services again.\nAfter connecting to NebulaGraph with NebulaGraph Console, run SHOW HOSTS GRAPH, SHOW HOSTS STORAGE, or SHOW HOSTS META to check the version of the responding service respectively.\nERROR: toomanyrequests when docker-compose pull\nYou may meet the following error.\nERROR: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit.\nYou have met the rate limit of Docker Hub. Learn more on Understanding Docker Hub Rate Limiting.\nHow to update the NebulaGraph Console client\nThe command docker-compose pull updates both the NebulaGraph services and the NebulaGraph Console.","url":"https://docs.nebula-graph.io/3.6.0/2.quick-start/1.quick-start-workflow/","type":"doc"},{"title":"Step 1: Install NebulaGraph","content":"RPM and DEB are common package formats on Linux systems. This topic shows how to quickly install NebulaGraph with the RPM or DEB package.\nPrerequisites\nThe tool wget is installed.\nStep 1: Download the package from cloud service\nDownload the released version.URL:\n//Centos 7\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.el7.x86_64.rpm\n//Centos 8\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.el8.x86_64.rpm\n//Ubuntu 1604\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.ubuntu1604.amd64.deb\n//Ubuntu 1804\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.ubuntu1804.amd64.deb\n//Ubuntu 2004\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.ubuntu2004.amd64.deb\nFor example, download the release package 3.6.0 for Centos 7.5:\nwget https://oss-cdn.nebula-graph.io/package/3.6.0/nebula-graph-3.6.0.el7.x86_64.rpm\nwget https://oss-cdn.nebula-graph.io/package/3.6.0/nebula-graph-3.6.0.el7.x86_64.rpm.sha256sum.txt\nDownload the release package 3.6.0 for Ubuntu 1804:\nwget https://oss-cdn.nebula-graph.io/package/3.6.0/nebula-graph-3.6.0.ubuntu1804.amd64.deb\nwget https://oss-cdn.nebula-graph.io/package/3.6.0/nebula-graph-3.6.0.ubuntu1804.amd64.deb.sha256sum.txt\nDownload the nightly version.\nURL:\n//Centos 7\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.el7.x86_64.rpm\n//Centos 8\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.el8.x86_64.rpm\n//Ubuntu 1604\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.ubuntu1604.amd64.deb\n//Ubuntu 1804\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.ubuntu1804.amd64.deb\n//Ubuntu 2004\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.ubuntu2004.amd64.deb\nFor example, download the Centos 7.5 package developed and built in 2021.11.28:\nwget https://oss-cdn.nebula-graph.io/package/nightly/2021.11.28/nebula-graph-2021.11.28-nightly.el7.x86_64.rpm\nwget https://oss-cdn.nebula-graph.io/package/nightly/2021.11.28/nebula-graph-2021.11.28-nightly.el7.x86_64.rpm.sha256sum.txt\nFor example, download the Ubuntu 1804 package developed and built in 2021.11.28:\nwget https://oss-cdn.nebula-graph.io/package/nightly/2021.11.28/nebula-graph-2021.11.28-nightly.ubuntu1804.amd64.deb\nwget https://oss-cdn.nebula-graph.io/package/nightly/2021.11.28/nebula-graph-2021.11.28-nightly.ubuntu1804.amd64.deb.sha256sum.txt\nStep 2: Install NebulaGraph\nUse the following syntax to install with an RPM package.\n$ sudo rpm -ivh --prefix= \nThe option --prefix indicates the installation path. The default path is /usr/local/nebula/.\nFor example, to install an RPM package in the default path for the 3.6.0 version, run the following command.\nsudo rpm -ivh nebula-graph-3.6.0.el7.x86_64.rpm\nUse the following syntax to install with a DEB package.\n$ sudo dpkg -i \nFor example, to install a DEB package for the 3.6.0 version, run the following command.\nsudo dpkg -i nebula-graph-3.6.0.ubuntu1804.amd64.deb\nNext to do\nStart NebulaGraph \nConnect to NebulaGraph","url":"https://docs.nebula-graph.io/3.6.0/2.quick-start/2.install-nebula-graph/","type":"doc"},{"title":"Step 2: Manage NebulaGraph Service","content":"NebulaGraph supports managing services with scripts. \nManage services with script\nYou can use the nebula.service script to start, stop, restart, terminate, and check the NebulaGraph services.\nSyntax\n$ sudo /usr/local/nebula/scripts/nebula.service\n[-v] [-c ]\n\n\nParameter\nDescription\n-v\nDisplay detailed debugging information.\n-c\nSpecify the configuration file path. The default path is /usr/local/nebula/etc/.\nstart\nStart the target services.\nstop\nStop the target services.\nrestart\nRestart the target services.\nkill\nTerminate the target services.\nstatus\nCheck the status of the target services.\nmetad\nSet the Meta Service as the target service.\ngraphd\nSet the Graph Service as the target service.\nstoraged\nSet the Storage Service as the target service.\nall\nSet all the NebulaGraph services as the target services.\nStart NebulaGraph\nRun the following command to start NebulaGraph.\n$ sudo /usr/local/nebula/scripts/nebula.service start all\n[INFO] Starting nebula-metad...\n[INFO] Done\n[INFO] Starting nebula-graphd...\n[INFO] Done\n[INFO] Starting nebula-storaged...\n[INFO] Done\nStop NebulaGraph\nRun the following command to stop NebulaGraph.\n$ sudo /usr/local/nebula/scripts/nebula.service stop all\n[INFO] Stopping nebula-metad...\n[INFO] Done\n[INFO] Stopping nebula-graphd...\n[INFO] Done\n[INFO] Stopping nebula-storaged...\n[INFO] Done\nCheck the service status\nRun the following command to check the service status of NebulaGraph.\n$ sudo /usr/local/nebula/scripts/nebula.service status all\nNebulaGraph is running normally if the following information is returned.\nINFO] nebula-metad(33fd35e): Running as 29020, Listening on 9559\n[INFO] nebula-graphd(33fd35e): Running as 29095, Listening on 9669\n[WARN] nebula-storaged after v3.0.0 will not start service until it is added to cluster.\n[WARN] See Manage Storage hosts:ADD HOSTS in https://docs.nebula-graph.io/\n[INFO] nebula-storaged(33fd35e): Running as 29147, Listening on 9779\nIf the returned result is similar to the following one, there is a problem. You may also go to the NebulaGraph community for help.[INFO] nebula-metad: Running as 25600, Listening on 9559\n[INFO] nebula-graphd: Exited\n[INFO] nebula-storaged: Running as 25646, Listening on 9779\nThe NebulaGraph services consist of the Meta Service, Graph Service, and Storage Service. The configuration files for all three services are stored in the /usr/local/nebula/etc/ directory by default. You can check the configuration files according to the returned result to troubleshoot problems.\nNext to do\nConnect to NebulaGraph","url":"https://docs.nebula-graph.io/3.6.0/2.quick-start/5.start-stop-service/","type":"doc"},{"title":"Step 3: Connect to NebulaGraph","content":"This topic provides basic instruction on how to use the native CLI client NebulaGraph Console to connect to NebulaGraph.\nNebulaGraph supports multiple types of clients, including a CLI client, a GUI client, and clients developed in popular programming languages. For more information, see the client list.\nPrerequisites\nYou have started NebulaGraph services.\nThe machine on which you plan to run NebulaGraph Console has network access to the Graph Service of NebulaGraph.\nThe NebulaGraph Console version is compatible with the NebulaGraph version.\nSteps\nOn the NebulaGraph Console releases page, select a NebulaGraph Console version and click Assets.\nIn the Assets area, find the correct binary file for the machine where you want to run NebulaGraph Console and download the file to the machine.\n(Optional) Rename the binary file to nebula-console for convenience.\nOn the machine to run NebulaGraph Console, grant the execute permission of the nebula-console binary file to the user.\n$ chmod 111 nebula-console\nIn the command line interface, change the working directory to the one where the nebula-console binary file is stored.\nRun the following command to connect to NebulaGraph.\nFor Linux or macOS:\n$ ./nebula-console -addr -port -u -p \n[-t 120] [-e \"nGQL_statement\" | -f filename.nGQL]\nFor Windows:\n> nebula-console.exe -addr -port -u -p \n[-t 120] [-e \"nGQL_statement\" | -f filename.nGQL]\nParameter descriptions are as follows:\nParameter\nDescription\n-h/-help\nShows the help menu.\n-addr/-address\nSets the IP (or hostname) of the Graph service. The default address is 127.0.0.1. \n-P/-port\nSets the port number of the graphd service. The default port number is 9669.\n-u/-user\nSets the username of your NebulaGraph account. Before enabling authentication, you can use any existing username. The default username is root.\n-p/-password\nSets the password of your NebulaGraph account. Before enabling authentication, you can use any characters as the password.\n-t/-timeout\nSets an integer-type timeout threshold of the connection. The unit is millisecond. The default value is 120.\n-e/-eval\nSets a string-type nGQL statement. The nGQL statement is executed once the connection succeeds. The connection stops after the result is returned.\n-f/-file\nSets the path of an nGQL file. The nGQL statements in the file are executed once the connection succeeds. The result will be returned and the connection stops then.\n-enable_ssl\nEnables SSL encryption when connecting to NebulaGraph.\n-ssl_root_ca_path\nSets the storage path of the certification authority file.\n-ssl_cert_path\nSets the storage path of the certificate file.\n-ssl_private_key_path\nSets the storage path of the private key file.\nFor information on more parameters, see the project repository.","url":"https://docs.nebula-graph.io/3.6.0/2.quick-start/3.connect-to-nebula-graph/","type":"doc"},{"title":"Register the Storage Service","content":"When connecting to NebulaGraph for the first time, you have to add the Storage hosts, and confirm that all the hosts are online.\nPrerequisites\nYou have connected to NebulaGraph.\nSteps\nAdd the Storage hosts.\nRun the following command to add hosts:\nADD HOSTS : [,: ...];\nExample：\n ADD HOSTS 192.168.10.100:9779, 192.168.10.101:9779, 192.168.10.102:9779;\nCheck the status of the hosts to make sure that they are all online.\n SHOW HOSTS;\nThe Status column of the result above shows that all Storage hosts are online.","url":"https://docs.nebula-graph.io/3.6.0/2.quick-start/3.1add-storage-hosts/","type":"doc"},{"title":"Step 4: Use nGQL (CRUD)","content":"This topic will describe the basic CRUD operations in NebulaGraph.\nFor more information, see nGQL guide.\nGraph space and NebulaGraph schema\nA NebulaGraph instance consists of one or more graph spaces. Graph spaces are physically isolated from each other. You can use different graph spaces in the same instance to store different datasets.\nTo insert data into a graph space, define a schema for the graph database. NebulaGraph schema is based on the following components.\nSchema component\nDescription\nVertex\nRepresents an entity in the real world. A vertex can have zero to multiple tags.\nTag\nThe type of the same group of vertices. It defines a set of properties that describes the types of vertices.\nEdge\nRepresents a directed relationship between two vertices.\nEdge type\nThe type of an edge. It defines a group of properties that describes the types of edges.\nFor more information, see Data modeling.\nIn this topic, we will use the following dataset to demonstrate basic CRUD operations.\nAsync implementation of CREATE and ALTER\nCREATE SPACE\nCREATE TAG\nCREATE EDGE\nALTER TAG\nALTER EDGE\nCREATE TAG INDEX\nCREATE EDGE INDEX\nCreate and use a graph space\nnGQL syntax\nCreate a graph space:CREATE SPACE [IF NOT EXISTS] (\n[partition_num = ,]\n[replica_factor = ,]\nvid_type = {FIXED_STRING() | INT64}\n[COMMENT = ''];\nFor more information on parameters, see CREATE SPACE.\nList graph spaces and check if the creation is successful: SHOW SPACES;\nUse a graph space:USE ;\nExamples\nUse the following statement to create a graph space named basketballplayer.\n CREATE SPACE basketballplayer(partition_num=15, replica_factor=1, vid_type=fixed_string(30));\nCheck the partition distribution with SHOW HOSTS to make sure that the partitions are distributed in a balanced way.\n SHOW HOSTS;\nIf the Leader distribution is uneven, use BALANCE LEADER to redistribute the partitions. For more information, see BALANCE.\nUse the basketballplayer graph space.\nnebula[(none)]> USE basketballplayer;\nYou can use SHOW SPACES to check the graph space you created.\n SHOW SPACES;\nCreate tags and edge types\nnGQL syntax\nCREATE {TAG | EDGE} [IF NOT EXISTS] { | }\n [NULL | NOT NULL] [DEFAULT ] [COMMENT '']\n [{, [NULL | NOT NULL] [DEFAULT ] [COMMENT '']} ...] \n [TTL_DURATION = ]\n [TTL_COL = ]\n [COMMENT = ''];\nFor more information on parameters, see CREATE TAG and CREATE EDGE.\nExamples\nCreate tags player and team, and edge types follow and serve. Descriptions are as follows.\nComponent name\nType\nProperty\nplayer\nTag\nname (string), age (int)\nteam\nTag\nname (string)\nfollow\nEdge type\ndegree (int)\nserve\nEdge type\nstart_year (int), end_year (int)\n CREATE TAG player(name string, age int);\n CREATE TAG team(name string);\n CREATE EDGE follow(degree int);\n CREATE EDGE serve(start_year int, end_year int);\nInsert vertices and edges\nYou can use the INSERT statement to insert vertices or edges based on existing tags or edge types.\nnGQL syntax\nInsert vertices:INSERT VERTEX [IF NOT EXISTS] [tag_props, [tag_props] ...]\nVALUES : ([prop_value_list])\ntag_props:\n tag_name ([prop_name_list])\nprop_name_list:\n [prop_name [, prop_name] ...]\nprop_value_list:\n [prop_value [, prop_value] ...] \nvid is short for Vertex ID. A vid must be a unique string value in a graph space. For details, see INSERT VERTEX.\nInsert edges:\nINSERT EDGE [IF NOT EXISTS] ( ) VALUES \n -> [@] : ( )\n[, -> [@] : ( ), ...];\n ::=\n[ [, ] ...]\n ::=\n[ [, ] ...]\nFor more information on parameters, see INSERT EDGE.\nExamples\nInsert vertices representing basketball players and teams: INSERT VERTEX player(name, age) VALUES \"player100\":(\"Tim Duncan\", 42);\n INSERT VERTEX player(name, age) VALUES \"player101\":(\"Tony Parker\", 36);\n INSERT VERTEX player(name, age) VALUES \"player102\":(\"LaMarcus Aldridge\", 33);\n INSERT VERTEX team(name) VALUES \"team203\":(\"Trail Blazers\"), \"team204\":(\"Spurs\");\nInsert edges representing the relations between basketball players and teams: INSERT EDGE follow(degree) VALUES \"player101\" -> \"player100\":(95);\n INSERT EDGE follow(degree) VALUES \"player101\" -> \"player102\":(90);\n INSERT EDGE follow(degree) VALUES \"player102\" -> \"player100\":(75);\n INSERT EDGE serve(start_year, end_year) VALUES \"player101\" -> \"team204\":(1999, 2018),\"player102\" -> \"team203\":(2006, 2015);\nRead data\nThe GO statement can traverse the database based on specific conditions. A GO traversal starts from one or more vertices, along one or more edges, and returns information in a form specified in the YIELD clause.\nThe FETCH statement is used to get properties from vertices or edges.\nThe LOOKUP statement is based on indexes. It is used together with the WHERE clause to search for the data that meet the specific conditions.\nThe MATCH statement is the most commonly used statement for graph data querying. It can describe all kinds of graph patterns, but it relies on indexes to match data patterns in NebulaGraph. Therefore, its performance still needs optimization.\nnGQL syntax\nGOGO [[ TO] {STEP|STEPS} ] FROM \nOVER [{REVERSELY | BIDIRECT}]\n[ WHERE ]\nYIELD [DISTINCT] \n[{ SAMPLE | }]\n[| GROUP BY { | expression> | } YIELD ]\n[| ORDER BY [{ASC | DESC}]]\n[| LIMIT [,] ];\nFETCH\nFetch properties on tags:\nFETCH PROP ON {[, tag_name ...] | *}\n [, vid ...]\nYIELD [AS ];\nFetch properties on edges:\nFETCH PROP ON -> [@] [, -> ...]\nYIELD ;\nLOOKUPLOOKUP ON { | }\n[WHERE [AND ...]]\nYIELD [AS ];\n\n [AS ] [, [AS ] ...];\nMATCHMATCH [] RETURN [];\nExamples of GO statement\nSearch for the players that the player with VID player101 follows. GO FROM \"player101\" OVER follow YIELD id($$);\nFilter the players that the player with VID player101 follows whose age is equal to or greater than 35. Rename the corresponding columns in the results with Teammate and Age. GO FROM \"player101\" OVER follow WHERE properties($$).age >= 35 \\\n YIELD properties($$).name AS Teammate, properties($$).age AS Age;\n |-------------+---------------------------------------------------------------------|\n | YIELD | Specifies what values or results you want to return from the query. |\n | $$ | Represents the target vertices. |\n | \\ | A line-breaker. |\nSearch for the players that the player with VID player101 follows. Then retrieve the teams of the players that the player with VID player100 follows. To combine the two queries, use a pipe or a temporary variable.\nWith a pipe:\n GO FROM \"player101\" OVER follow YIELD dst(edge) AS id | \\\n GO FROM $-.id OVER serve YIELD properties($$).name AS Team, \\\n properties($^).name AS Player;\nClause/Sign\nDescription\n$^\nRepresents the source vertex of the edge.\nA pipe symbol can combine multiple queries.\n$-\nRepresents the outputs of the query before the pipe symbol.\nWith a temporary variable:\n $var = GO FROM \"player101\" OVER follow YIELD dst(edge) AS id; \\\n GO FROM $var.id OVER serve YIELD properties($$).name AS Team, \\\n properties($^).name AS Player;\nExample of FETCH statement\nUse FETCH: Fetch the properties of the player with VID player100.\n FETCH PROP ON player \"player100\" YIELD properties(vertex);\nUpdate vertices and edges\nUsers can use the UPDATE or the UPSERT statements to update existing data.\nUPSERT is the combination of UPDATE and INSERT. If you update a vertex or an edge with UPSERT, the database will insert a new vertex or edge if it does not exist.\nnGQL syntax\nUPDATE vertices:UPDATE VERTEX SET \n[WHEN ] [YIELD ];\nUPDATE edges:UPDATE EDGE ON -> [@rank] \nSET [WHEN ] [YIELD ];\nUPSERT vertices or edges:UPSERT {VERTEX | EDGE } SET \n[WHEN ] [YIELD ];\nExamples\nUPDATE the name property of the vertex with VID player100 and check the result with the FETCH statement. UPDATE VERTEX \"player100\" SET player.name = \"Tim\";\n FETCH PROP ON player \"player100\" YIELD properties(vertex);\nUPDATE the degree property of an edge and check the result with the FETCH statement. UPDATE EDGE ON follow \"player101\" -> \"player100\" SET degree = 96;\n FETCH PROP ON follow \"player101\" -> \"player100\" YIELD properties(edge);\nInsert a vertex with VID player111 and UPSERT it. INSERT VERTEX player(name,age) VALUES \"player111\":(\"David West\", 38);\n UPSERT VERTEX \"player111\" SET player.name = \"David\", player.age = $^.player.age + 11 \\\n WHEN $^.player.name == \"David West\" AND $^.player.age > 20 \\\n YIELD $^.player.name AS Name, $^.player.age AS Age;\nDelete vertices and edges\nnGQL syntax\nDelete vertices:DELETE VERTEX [, ...]\nDelete edges:DELETE EDGE -> [@]\n[, -> ...]\nExamples\nDelete vertices: DELETE VERTEX \"player111\", \"team203\";\nDelete edges: DELETE EDGE follow \"player101\" -> \"team204\";\nAbout indexes\nUsers can add indexes to tags and edge types with the CREATE INDEX statement.\nnGQL syntax\nCreate an index:CREATE {TAG | EDGE} INDEX [IF NOT EXISTS] \nON { | } ([]) [COMMENT = ''];\nRebuild an index:REBUILD {TAG | EDGE} INDEX ;\nExamples of LOOKUP and MATCH (index-based)\nMake sure there is an index for LOOKUP or MATCH to use. If there is not, create an index first.\nFind the information of the vertex with the tag player and its value of the name property is Tony Parker.\nThis example creates the index player_index_1 on the name property.\n CREATE TAG INDEX IF NOT EXISTS player_index_1 ON player(name(20));\nThis example rebuilds the index to make sure it takes effect on pre-existing data.\n REBUILD TAG INDEX player_index_1\nThis example uses the LOOKUP statement to retrieve the vertex property.\n LOOKUP ON player WHERE player.name == \"Tony Parker\" \\\n YIELD properties(vertex).name AS name, properties(vertex).age AS age;\nThis example uses the MATCH statement to retrieve the vertex property.\n MATCH (v:player{name:\"Tony Parker\"}) RETURN v;","url":"https://docs.nebula-graph.io/3.6.0/2.quick-start/4.nebula-graph-crud/","type":"doc"},{"title":"nGQL cheatsheet","content":"Functions\nMath functions\nFunction\nDescription\ndouble abs(double x)\nReturns the absolute value of the argument.\ndouble floor(double x)\nReturns the largest integer value smaller than or equal to the argument. (Rounds down)\ndouble ceil(double x)\nReturns the smallest integer greater than or equal to the argument. (Rounds up)\ndouble round(double x)\nReturns the integer value nearest to the argument. Returns a number farther away from 0 if the argument is in the middle.\ndouble sqrt(double x)\nReturns the square root of the argument.\ndouble cbrt(double x)\nReturns the cubic root of the argument.\ndouble hypot(double x, double y)\nReturns the hypotenuse of a right-angled triangle.\ndouble pow(double x, double y)\nReturns the result of xy.\ndouble exp(double x)\nReturns the result of ex.\ndouble exp2(double x)\nReturns the result of 2x.\ndouble log(double x)\nReturns the base-e logarithm of the argument.\ndouble log2(double x)\nReturns the base-2 logarithm of the argument.\ndouble log10(double x)\nReturns the base-10 logarithm of the argument.\ndouble sin(double x)\nReturns the sine of the argument.\ndouble asin(double x)\nReturns the inverse sine of the argument.\ndouble cos(double x)\nReturns the cosine of the argument.\ndouble acos(double x)\nReturns the inverse cosine of the argument.\ndouble tan(double x)\nReturns the tangent of the argument.\ndouble atan(double x)\nReturns the inverse tangent of the argument.\ndouble rand()\nReturns a random floating point number in the range from 0 (inclusive) to 1 (exclusive); i.e.[0,1).\nint rand32(int min, int max)\nReturns a random 32-bit integer in [min, max).If you set only one argument, it is parsed as max and min is 0 by default.If you set no argument, the system returns a random signed 32-bit integer.\nint rand64(int min, int max)\nReturns a random 64-bit integer in [min, max).If you set only one argument, it is parsed as max and min is 0 by default.If you set no argument, the system returns a random signed 64-bit integer.\nbit_and()\nBitwise AND.\nbit_or()\nBitwise OR.\nbit_xor()\nBitwise XOR.\nint size()\nReturns the number of elements in a list or a map or the length of a string.\nint range(int start, int end, int step)\nReturns a list of integers from [start,end] in the specified steps. step is 1 by default.\nint sign(double x)\nReturns the signum of the given number.If the number is 0, the system returns 0.If the number is negative, the system returns -1.If the number is positive, the system returns 1.\ndouble e()\nReturns the base of the natural logarithm, e (2.718281828459045).\ndouble pi()\nReturns the mathematical constant pi (3.141592653589793).\ndouble radians()\nConverts degrees to radians. radians(180) returns 3.141592653589793.\nAggregating functions\nFunction\nDescription\navg()\nReturns the average value of the argument.\ncount()\nSyntax: count({expr | *}) .count()returns the number of rows (including NULL). count(expr)returns the number of non-NULL values that meet the expression. count() and size() are different.\nmax()\nReturns the maximum value.\nmin()\nReturns the minimum value.\ncollect()\nThe collect() function returns a list containing the values returned by an expression. Using this function aggregates data by merging multiple records or values into a single list.\nstd()\nReturns the population standard deviation.\nsum()\nReturns the sum value.\nString functions\nFunction\nDescription\nint strcasecmp(string a, string b)\nCompares string a and b without case sensitivity. When a = b, the return\nstring lower(string a)\nReturns the argument in lowercase.\nstring toLower(string a)\nThe same as lower().\nstring upper(string a)\nReturns the argument in uppercase.\nstring toUpper(string a)\nThe same as upper().\nint length(a)\nReturns the length of the given string in bytes or the length of a path in hops.\nstring trim(string a)\nRemoves leading and trailing spaces.\nstring ltrim(string a)\nRemoves leading spaces.\nstring rtrim(string a)\nRemoves trailing spaces.\nstring left(string a, int count)\nReturns a substring consisting of count characters from the left side of\nstring right(string a, int count)\nReturns a substring consisting of count characters from the right side of\nstring lpad(string a, int size, string letters)\nLeft-pads string a with string letters and returns a\nstring rpad(string a, int size, string letters)\nRight-pads string a with string letters and returns a\nstring substr(string a, int pos, int count)\nReturns a substring extracting count characters starting from\nstring substring(string a, int pos, int count)\nThe same as substr().\nstring reverse(string)\nReturns a string in reverse order.\nstring replace(string a, string b, string c)\nReplaces string b in string a with string c.\nlist split(string a, string b)\nSplits string a at string b and returns a list of strings.\nconcat()\nThe concat() function requires at least two or more strings. All the parameters are concatenated into one string.Syntax: concat(string1,string2,...)\nconcat_ws()\nThe concat_ws() function connects two or more strings with a predefined separator.\nextract()\nextract() uses regular expression matching to retrieve a single substring or all substrings from a string.\njson_extract()\nThe json_extract() function converts the specified JSON string to map.\nData and time functions\nFunction\nDescription\nint now()\nReturns the current timestamp of the system.\ntimestamp timestamp()\nReturns the current timestamp of the system.\ndate date()\nReturns the current UTC date based on the current system.\ntime time()\nReturns the current UTC time based on the current system.\ndatetime datetime()\nReturns the current UTC date and time based on the current system.\nSchema-related functions\nFor nGQL statements\nFunction\nDescription\nid(vertex)\nReturns the ID of a vertex. The data type of the result is the same as the vertex ID.\nmap properties(vertex)\nReturns the properties of a vertex.\nmap properties(edge)\nReturns the properties of an edge.\nstring type(edge)\nReturns the edge type of an edge.\nsrc(edge)\nReturns the source vertex ID of an edge. The data type of the result is the same as the vertex ID.\ndst(edge)\nReturns the destination vertex ID of an edge. The data type of the result is the same as the vertex ID.\nint rank(edge)\nReturns the rank value of an edge.\nvertex\nReturns the information of vertices, including VIDs, tags, properties, and values.\nedge\nReturns the information of edges, including edge types, source vertices, destination vertices, ranks, properties, and values.\nvertices\nReturns the information of vertices in a subgraph. For more information, see GET SUBGRAPH.\nedges\nReturns the information of edges in a subgraph. For more information, see GET SUBGRAPH.\npath\nReturns the information of a path. For more information, see FIND PATH.\nFor statements compatible with openCypher\nFunction\nDescription\nid()\nReturns the ID of a vertex. The data type of the result is the same as the vertex ID.\nlist tags()\nReturns the Tag of a vertex, which serves the same purpose as labels().\nlist labels()\nReturns the Tag of a vertex, which serves the same purpose as tags(). This function is used for compatibility with openCypher syntax.\nmap properties()\nReturns the properties of a vertex or an edge.\nstring type()\nReturns the edge type of an edge.\nsrc()\nReturns the source vertex ID of an edge. The data type of the result is the same as the vertex ID.\ndst()\nReturns the destination vertex ID of an edge. The data type of the result is the same as the vertex ID.\nvertex startNode()\nVisits an edge or a path and returns its source vertex ID.\nstring endNode()\nVisits an edge or a path and returns its destination vertex ID.\nint rank()\nReturns the rank value of an edge.\nList functions\nFunction\nDescription\nkeys(expr)\nReturns a list containing the string representations for all the property names of vertices, edges, or maps.\nlabels(vertex)\nReturns the list containing all the tags of a vertex.\nnodes(path)\nReturns the list containing all the vertices in a path.\nrange(start, end [, step])\nReturns the list containing all the fixed-length steps in [start,end]. step is 1 by default.\nrelationships(path)\nReturns the list containing all the relationships in a path.\nreverse(list)\nReturns the list reversing the order of all elements in the original list.\ntail(list)\nReturns all the elements of the original list, excluding the first one.\nhead(list)\nReturns the first element of a list.\nlast(list)\nReturns the last element of a list.\nreduce()\nThe reduce() function applies an expression to each element in a list one by one, chains the result to the next iteration by taking it as the initial value, and returns the final result.\nType conversion functions\nFunction\nDescription\nbool toBoolean()\nConverts a string value to a boolean value.\nfloat toFloat()\nConverts an integer or string value to a floating point number.\nstring toString()\nConverts non-compound types of data, such as numbers, booleans, and so on, to strings.\nint toInteger()\nConverts a floating point or string value to an integer value.\nset toSet()\nConverts a list or set value to a set value.\nint hash()\nThe hash() function returns the hash value of the argument. The argument can be a number, a string, a list, a boolean, null, or an expression that evaluates to a value of the preceding data types.\nPredicate functions\nPredicate functions return true or false. They are most commonly used in WHERE clauses.\n( IN WHERE )\nFunction\nDescription\nexists()\nReturns true if the specified property exists in the vertex, edge or map. Otherwise, returns false.\nany()\nReturns true if the specified predicate holds for at least one element in the given list. Otherwise, returns false.\nall()\nReturns true if the specified predicate holds for all elements in the given list. Otherwise, returns false.\nnone()\nReturns true if the specified predicate holds for no element in the given list. Otherwise, returns false.\nsingle()\nReturns true if the specified predicate holds for exactly one of the elements in the given list. Otherwise, returns false.\nConditional expressions functions\nFunction\nDescription\nCASE\nThe CASE expression uses conditions to filter the result of an nGQL query statement. It is usually used in the YIELD and RETURN clauses. The CASE expression will traverse all the conditions. When the first condition is met, the CASE expression stops reading the conditions and returns the result. If no conditions are met, it returns the result in the ELSE clause. If there is no ELSE clause and no conditions are met, it returns NULL.\ncoalesce()\nReturns the first not null value in all expressions.\nGeneral queries statements\nMATCH\nMATCH [] RETURN [];\nPattern\nExample\nDescription\nMatch vertices\n(v)\nYou can use a user-defined variable in a pair of parentheses to represent a vertex in a pattern. For example: (v).\nMatch tags\nMATCH (v:player) RETURN v\nYou can specify a tag with : after the vertex in a pattern.\nMatch multiple tags\nMATCH (v:player:team) RETURN v\nTo match vertices with multiple tags, use colons (:).\nMatch vertex properties\nMATCH (v:player{name:\"Tim Duncan\"}) RETURN v MATCH (v) WITH v, properties(v) as props, keys(properties(v)) as kk WHERE [i in kk where props[i] == \"Tim Duncan\"] RETURN v\nYou can specify a vertex property with {: } after the tag in a pattern; or use a vertex property value to get vertices directly.\nMatch a VID.\nMATCH (v) WHERE id(v) == 'player101' RETURN v\nYou can use the VID to match a vertex. The id() function can retrieve the VID of a vertex.\nMatch multiple VIDs.\nMATCH (v:player { name: 'Tim Duncan' })--(v2) WHERE id(v2) IN [\"player101\", \"player102\"] RETURN v2\nTo match multiple VIDs, use WHERE id(v) IN [vid_list].\nMatch connected vertices\nMATCH (v:player{name:\"Tim Duncan\"})--(v2) RETURN v2.player.name AS Name\nYou can use the -- symbol to represent edges of both directions and match vertices connected by these edges. You can add a > or < to the -- symbol to specify the direction of an edge.\nMatch paths\nMATCH p=(v:player{name:\"Tim Duncan\"})-->(v2) RETURN p\nConnected vertices and edges form a path. You can use a user-defined variable to name a path as follows.\nMatch edges\nMATCH (v:player{name:\"Tim Duncan\"})-[e]-(v2) RETURN eMATCH ()<-[e]-() RETURN e\nBesides using --, -->, or <-- to indicate a nameless edge, you can use a user-defined variable in a pair of square brackets to represent a named edge. For example: -[e]-.\nMatch an edge type\nMATCH ()-[e:follow]-() RETURN e\nJust like vertices, you can specify an edge type with : in a pattern. For example: -[e:follow]-.\nMatch edge type properties\nMATCH (v:player{name:\"Tim Duncan\"})-[e:follow{degree:95}]->(v2) RETURN e MATCH ()-[e]->() WITH e, properties(e) as props, keys(properties(e)) as kk WHERE [i in kk where props[i] == 90] RETURN e\nYou can specify edge type properties with {: } in a pattern. For example: [e:follow{likeness:95}]; or use an edge type property value to get edges directly.\nMatch multiple edge types\nMATCH (v:player{name:\"Tim Duncan\"})-[e:follow | :serve]->(v2) RETURN e\nThe | symbol can help matching multiple edge types. For example: [e:follow|:serve]. The English colon (:) before the first edge type cannot be omitted, but the English colon before the subsequent edge type can be omitted, such as [e:follow|serve].\nMatch multiple edges\nMATCH (v:player{name:\"Tim Duncan\"})-[]->(v2)<-[e:serve]-(v3) RETURN v2, v3\nYou can extend a pattern to match multiple edges in a path.\nMatch fixed-length paths\nMATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow*2]->(v2) RETURN DISTINCT v2 AS Friends\nYou can use the :* pattern to match a fixed-length path. hop must be a non-negative integer. The data type of e is the list.\nMatch variable-length paths\nMATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow*1..3]->(v2) RETURN v2 AS Friends\nminHop: Optional. It represents the minimum length of the path. minHop: must be a non-negative integer. The default value is 1.minHop and maxHop are optional and the default value is 1 and infinity respectively. The data type of e is the list.\nMatch variable-length paths with multiple edge types\nMATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow | serve*2]->(v2) RETURN DISTINCT v2\nYou can specify multiple edge types in a fixed-length or variable-length pattern. In this case, hop, minHop, and maxHop take effect on all edge types. The data type of e is the list.\nRetrieve vertex or edge information\nMATCH (v:player{name:\"Tim Duncan\"}) RETURN vMATCH (v:player{name:\"Tim Duncan\"})-[e]->(v2) RETURN e\nUse RETURN { | } to retrieve all the information of a vertex or an edge.\nRetrieve VIDs\nMATCH (v:player{name:\"Tim Duncan\"}) RETURN id(v)\nUse the id() function to retrieve VIDs.\nRetrieve tags\nMATCH (v:player{name:\"Tim Duncan\"}) RETURN labels(v)\nUse the labels() function to retrieve the list of tags on a vertex.To retrieve the nth element in the labels(v) list, use labels(v)[n-1].\nRetrieve a single property on a vertex or an edge\nMATCH (v:player{name:\"Tim Duncan\"}) RETURN v.player.age\nUse RETURN { | }. to retrieve a single property.Use AS to specify an alias for a property.\nRetrieve all properties on a vertex or an edge\nMATCH p=(v:player{name:\"Tim Duncan\"})-[]->(v2) RETURN properties(v2)\nUse the properties() function to retrieve all properties on a vertex or an edge.\nRetrieve edge types\nMATCH p=(v:player{name:\"Tim Duncan\"})-[e]->() RETURN DISTINCT type(e)\nUse the type() function to retrieve the matched edge types.\nRetrieve paths\nMATCH p=(v:player{name:\"Tim Duncan\"})-[*3]->() RETURN p\nUse RETURN to retrieve all the information of the matched paths.\nRetrieve vertices in a path\nMATCH p=(v:player{name:\"Tim Duncan\"})-[]->(v2) RETURN nodes(p)\nUse the nodes() function to retrieve all vertices in a path.\nRetrieve edges in a path\nMATCH p=(v:player{name:\"Tim Duncan\"})-[]->(v2) RETURN relationships(p)\nUse the relationships() function to retrieve all edges in a path.\nRetrieve path length\nMATCH p=(v:player{name:\"Tim Duncan\"})-[*..2]->(v2) RETURN p AS Paths, length(p) AS Length\nUse the length() function to retrieve the length of a path.\nOPTIONAL MATCH\nPattern\nExample\nDescription\nMatches patterns against your graph database, just like MATCH does.\nMATCH (m)-[]->(n) WHERE id(m)==\"player100\" OPTIONAL MATCH (n)-[]->(l) RETURN id(m),id(n),id(l)\nIf no matches are found, OPTIONAL MATCH will use a null for missing parts of the pattern.\nLOOKUP\nLOOKUP ON { | } \n[WHERE [AND ...]] \nYIELD [AS ]\nPattern\nExample\nDescription\nRetrieve vertices\nLOOKUP ON player WHERE player.name == \"Tony Parker\" YIELD player.name AS name, player.age AS age\nThe following example returns vertices whose name is Tony Parker and the tag is player.\nRetrieve edges\nLOOKUP ON follow WHERE follow.degree == 90 YIELD follow.degree\nReturns edges whose degree is 90 and the edge type is follow.\nList vertices with a tag\nLOOKUP ON player YIELD properties(vertex),id(vertex)\nShows how to retrieve the VID of all vertices tagged with player.\nList edges with an edge types\nLOOKUP ON follow YIELD edge AS e\nShows how to retrieve the source Vertex IDs, destination vertex IDs, and ranks of all edges of the follow edge type.\nCount the numbers of vertices or edges\nLOOKUP ON player YIELD id(vertex)| YIELD COUNT(*) AS Player_Count\nShows how to count the number of vertices tagged with player.\nCount the numbers of edges\nLOOKUP ON follow YIELD edge as e| YIELD COUNT(*) AS Like_Count\nShows how to count the number of edges of the follow edge type.\nGO\nGO [[ TO] {STEP|STEPS} ] FROM \nOVER [{REVERSELY | BIDIRECT}]\n[ WHERE ]\nYIELD [DISTINCT] \n[{SAMPLE | LIMIT }]\n[| GROUP BY {col_name | expr | position} YIELD ]\n[| ORDER BY [{ASC | DESC}]]\n[| LIMIT [,] ]\nExample\nDescription\nGO FROM \"player102\" OVER serve YIELD dst(edge)\nReturns the teams that player 102 serves.\nGO 2 STEPS FROM \"player102\" OVER follow YIELD dst(edge)\nReturns the friends of player 102 with 2 hops.\nGO FROM \"player100\", \"player102\" OVER serve WHERE properties(edge).start_year > 1995 YIELD DISTINCT properties($$).name AS team_name, properties(edge).start_year AS start_year, properties($^).name AS player_name\nAdds a filter for the traversal.\nGO FROM \"player100\" OVER follow, serve YIELD properties(edge).degree, properties(edge).start_year\nThe following example traverses along with multiple edge types. If there is no value for a property, the output is NULL.\nGO FROM \"player100\" OVER follow REVERSELY YIELD src(edge) AS destination\nThe following example returns the neighbor vertices in the incoming direction of player 100.\nGO FROM \"player100\" OVER follow REVERSELY YIELD src(edge) AS id | GO FROM $-.id OVER serve WHERE properties($^).age > 20 YIELD properties($^).name AS FriendOf, properties($$).name AS Team\nThe following example retrieves the friends of player 100 and the teams that they serve.\nGO FROM \"player102\" OVER follow YIELD dst(edge) AS both\nThe following example returns all the neighbor vertices of player 102.\nGO 2 STEPS FROM \"player100\" OVER follow YIELD src(edge) AS src, dst(edge) AS dst, properties($$).age AS age | GROUP BY $-.dst YIELD $-.dst AS dst, collect_set($-.src) AS src, collect($-.age) AS age\nThe following example the outputs according to age.\nFETCH\nFetch vertex properties\nFETCH PROP ON {[, tag_name ...] | *} \n [, vid ...] \nYIELD [AS ]\nExample\nDescription\nFETCH PROP ON player \"player100\" YIELD properties(vertex)\nSpecify a tag in the FETCH statement to fetch the vertex properties by that tag.\nFETCH PROP ON player \"player100\" YIELD player.name AS name\nUse a YIELD clause to specify the properties to be returned.\nFETCH PROP ON player \"player101\", \"player102\", \"player103\" YIELD properties(vertex)\nSpecify multiple VIDs (vertex IDs) to fetch properties of multiple vertices. Separate the VIDs with commas.\nFETCH PROP ON player, t1 \"player100\", \"player103\" YIELD properties(vertex)\nSpecify multiple tags in the FETCH statement to fetch the vertex properties by the tags. Separate the tags with commas.\nFETCH PROP ON * \"player100\", \"player106\", \"team200\" YIELD properties(vertex)\nSet an asterisk symbol * to fetch properties by all tags in the current graph space.\nFetch edge properties\nFETCH PROP ON -> [@] [, -> ...]\nYIELD ;\nExample\nDescription\nFETCH PROP ON serve \"player100\" -> \"team204\" YIELD properties(edge)\nThe following statement fetches all the properties of the serve edge that connects vertex \"player100\" and vertex \"team204\".\nFETCH PROP ON serve \"player100\" -> \"team204\" YIELD serve.start_year\nUse a YIELD clause to fetch specific properties of an edge.\nFETCH PROP ON serve \"player100\" -> \"team204\", \"player133\" -> \"team202\" YIELD properties(edge)\nSpecify multiple edge patterns ( -> [@]) to fetch properties of multiple edges. Separate the edge patterns with commas.\nFETCH PROP ON serve \"player100\" -> \"team204\"@1 YIELD properties(edge)\nTo fetch on an edge whose rank is not 0, set its rank in the FETCH statement.\nGO FROM \"player101\" OVER follow YIELD follow._src AS s, follow._dst AS d | FETCH PROP ON follow $-.s -> $-.d YIELD follow.degree\nThe following statement returns the degree values of the follow edges that start from vertex \"player101\".\n$var = GO FROM \"player101\" OVER follow YIELD follow._src AS s, follow._dst AS d; FETCH PROP ON follow $var.s -> $var.d YIELD follow.degree\nYou can use user-defined variables to construct similar queries.\nSHOW\nStatement\nSyntax\nExample\nDescription\nSHOW CHARSET\nSHOW CHARSET\nSHOW CHARSET\nShows the available character sets.\nSHOW COLLATION\nSHOW COLLATION\nSHOW COLLATION\nShows the collations supported by NebulaGraph.\nSHOW CREATE SPACE\nSHOW CREATE SPACE \nSHOW CREATE SPACE basketballplayer\nShows the creating statement of the specified graph space.\nSHOW CREATE TAG/EDGE\nSHOW CREATE {TAG | EDGE }\nSHOW CREATE TAG player\nShows the basic information of the specified tag.\nSHOW HOSTS\nSHOW HOSTS [GRAPH | STORAGE | META]\nSHOW HOSTSSHOW HOSTS GRAPH\nShows the host and version information of Graph Service, Storage Service, and Meta Service.\nSHOW INDEX STATUS\nSHOW {TAG | EDGE} INDEX STATUS\nSHOW TAG INDEX STATUS\nShows the status of jobs that rebuild native indexes, which helps check whether a native index is successfully rebuilt or not.\nSHOW INDEXES\nSHOW {TAG | EDGE} INDEXES\nSHOW TAG INDEXES\nShows the names of existing native indexes.\nSHOW PARTS\nSHOW PARTS []\nSHOW PARTS\nShows the information of a specified partition or all partitions in a graph space.\nSHOW ROLES\nSHOW ROLES IN \nSHOW ROLES in basketballplayer\nShows the roles that are assigned to a user account.\nSHOW SNAPSHOTS\nSHOW SNAPSHOTS\nSHOW SNAPSHOTS\nShows the information of all the snapshots.\nSHOW SPACES\nSHOW SPACES\nSHOW SPACES\nShows existing graph spaces in NebulaGraph.\nSHOW STATS\nSHOW STATS\nSHOW STATS\nShows the statistics of the graph space collected by the latest STATS job.\nSHOW TAGS/EDGES\nSHOW TAGS | EDGES\nSHOW TAGS,SHOW EDGES\nShows all the tags in the current graph space.\nSHOW USERS\nSHOW USERS\nSHOW USERS\nShows the user information.\nSHOW SESSIONS\nSHOW SESSIONS\nSHOW SESSIONS\nShows the information of all the sessions.\nSHOW SESSIONS\nSHOW SESSION \nSHOW SESSION 1623304491050858\nShows a specified session with its ID.\nSHOW QUERIES\nSHOW [ALL] QUERIES\nSHOW QUERIES\nShows the information of working queries in the current session.\nSHOW META LEADER\nSHOW META LEADER\nSHOW META LEADER\nShows the information of the leader in the current Meta cluster.\nClauses and options\nClause\nSyntax\nExample\nDescription\nGROUP BY\nGROUP BY YIELD , \nGO FROM \"player100\" OVER follow BIDIRECT YIELD $$.player.name as Name | GROUP BY $-.Name YIELD $-.Name as Player, count(*) AS Name_Count\nFinds all the vertices connected directly to vertex \"player100\", groups the result set by player names, and counts how many times the name shows up in the result set.\nLIMIT\nYIELD [| LIMIT [,] ]\nGO FROM \"player100\" OVER follow REVERSELY YIELD $$.player.name AS Friend, $$.player.age AS Age | ORDER BY $-.Age, $-.Friend | LIMIT 1, 3\nReturns the 3 rows of data starting from the second row of the sorted output.\nSKIP\nRETURN [SKIP ] [LIMIT ]\nMATCH (v:player{name:\"Tim Duncan\"}) --> (v2) RETURN v2.player.name AS Name, v2.player.age AS Age ORDER BY Age DESC SKIP 1\nSKIP can be used alone to set the offset and return the data after the specified position.\nSAMPLE\n SAMPLE ;\nGO 3 STEPS FROM \"player100\" OVER * YIELD properties($$).name AS NAME, properties($$).age AS Age SAMPLE [1,2,3];\nTakes samples evenly in the result set and returns the specified amount of data.\nORDER BY\n ORDER BY [ASC | DESC] [, [ASC | DESC] ...]\nFETCH PROP ON player \"player100\", \"player101\", \"player102\", \"player103\" YIELD player.age AS age, player.name AS name | ORDER BY $-.age ASC, $-.name DESC\nThe ORDER BY clause specifies the order of the rows in the output.\nRETURN\nRETURN {||.|.|...}\nMATCH (v:player) RETURN v.player.name, v.player.age LIMIT 3\nReturns the first three rows with values of the vertex properties name and age.\nTTL\nCREATE TAG ( , , ...) ttl_duration= , ttl_col = \nCREATE TAG t2(a int, b int, c string) ttl_duration= 100, ttl_col = \"a\"\nCreate a tag and set the TTL options.\nWHERE\nWHERE {. {>|==|<|...} ...}\nMATCH (v:player) WHERE v.player.name == \"Tim Duncan\" XOR (v.player.age < 30 AND v.player.name == \"Yao Ming\") OR NOT (v.player.name == \"Yao Ming\" OR v.player.name == \"Tim Duncan\") RETURN v.player.name, v.player.age\nThe WHERE clause filters the output by conditions. The WHERE clause usually works in Native nGQL GO and LOOKUP statements, and OpenCypher MATCH and WITH statements.\nYIELD\nYIELD [DISTINCT] [AS ] [, [AS ] ...] [WHERE ];\nGO FROM \"player100\" OVER follow YIELD dst(edge) AS ID | FETCH PROP ON player $-.ID YIELD player.age AS Age | YIELD AVG($-.Age) as Avg_age, count(*)as Num_friends\nFinds the players that \"player100\" follows and calculates their average age.\nWITH\nMATCH $expressions WITH {nodes()|labels()|...}\nMATCH p=(v:player{name:\"Tim Duncan\"})--() WITH nodes(p) AS n UNWIND n AS n1 RETURN DISTINCT n1\nThe WITH clause can retrieve the output from a query part, process it, and pass it to the next query part as the input.\nUNWIND\nUNWIND AS \nUNWIND [1,2,3] AS n RETURN n\nSplits a list into rows.\nSpace statements\nStatement\nSyntax\nExample\nDescription\nCREATE SPACE\nCREATE SPACE [IF NOT EXISTS] ( [partition_num = ,] [replica_factor = ,] vid_type = {FIXED_STRING() | INT[64]} ) [COMMENT = '']\nCREATE SPACE my_space_1 (vid_type=FIXED_STRING(30))\nCreates a graph space with\nCREATE SPACE\nCREATE SPACE AS \nCREATE SPACE my_space_4 as my_space_3\nClone a graph. space.\nUSE\nUSE \nUSE space1\nSpecifies a graph space as the current working graph space for subsequent queries.\nSHOW SPACES\nSHOW SPACES\nSHOW SPACES\nLists all the graph spaces in the NebulaGraph examples.\nDESCRIBE SPACE\nDESC[RIBE] SPACE \nDESCRIBE SPACE basketballplayer\nReturns the information about the specified graph space.\nCLEAR SPACE\nCLEAR SPACE [IF EXISTS] \nDeletes the vertices and edges in a graph space, but does not delete the graph space itself and the schema information.\nDROP SPACE\nDROP SPACE [IF EXISTS] \nDROP SPACE basketballplayer\nDeletes everything in the specified graph space.\nTAG statements\nStatement\nSyntax\nExample\nDescription\nCREATE TAG\nCREATE TAG [IF NOT EXISTS] ( [NULL | NOT NULL] [DEFAULT ] [COMMENT ''] [{, [NULL | NOT NULL] [DEFAULT ] [COMMENT '']} ...] ) [TTL_DURATION = ] [TTL_COL = ] [COMMENT = '']\nCREATE TAG woman(name string, age int, married bool, salary double, create_time timestamp) TTL_DURATION = 100, TTL_COL = \"create_time\"\nCreates a tag with the given name in a graph space.\nDROP TAG\nDROP TAG [IF EXISTS] \nDROP TAG test;\nDrops a tag with the given name in the current working graph space.\nALTER TAG\nALTER TAG [, alter_definition] ...] [ttl_definition [, ttl_definition] ... ] [COMMENT = '']\nALTER TAG t1 ADD (p3 int, p4 string)\nAlters the structure of a tag with the given name in a graph space. You can add or drop properties, and change the data type of an existing property. You can also set a TTL (Time-To-Live) on a property, or change its TTL duration.\nSHOW TAGS\nSHOW TAGS\nSHOW TAGS\nShows the name of all tags in the current graph space.\nDESCRIBE TAG\nDESC[RIBE] TAG \nDESCRIBE TAG player\nReturns the information about a tag with the given name in a graph space, such as field names, data type, and so on.\nDELETE TAG\nDELETE TAG FROM \nDELETE TAG test1 FROM \"test\"\nDeletes a tag with the given name on a specified vertex.\nEdge type statements\nStatement\nSyntax\nExample\nDescription\nCREATE EDGE\nCREATE EDGE [IF NOT EXISTS] ( [NULL | NOT NULL] [DEFAULT ] [COMMENT ''] [{, [NULL | NOT NULL] [DEFAULT ] [COMMENT '']} ...] ) [TTL_DURATION = ] [TTL_COL = ] [COMMENT = '']\nCREATE EDGE e1(p1 string, p2 int, p3 timestamp) TTL_DURATION = 100, TTL_COL = \"p2\"\nCreates an edge type with the given name in a graph space.\nDROP EDGE\nDROP EDGE [IF EXISTS] \nDROP EDGE e1\nDrops an edge type with the given name in a graph space.\nALTER EDGE\nALTER EDGE [, alter_definition] ...] [ttl_definition [, ttl_definition] ... ] [COMMENT = '']\nALTER EDGE e1 ADD (p3 int, p4 string)\nAlters the structure of an edge type with the given name in a graph space.\nSHOW EDGES\nSHOW EDGES\nSHOW EDGES\nShows all edge types in the current graph space.\nDESCRIBE EDGE\nDESC[RIBE] EDGE \nDESCRIBE EDGE follow\nReturns the information about an edge type with the given name in a graph space, such as field names, data type, and so on.\nVertex statements\nStatement\nSyntax\nExample\nDescription\nINSERT VERTEX\nINSERT VERTEX [IF NOT EXISTS] [tag_props, [tag_props] ...] VALUES : ([prop_value_list])\nINSERT VERTEX t2 (name, age) VALUES \"13\":(\"n3\", 12), \"14\":(\"n4\", 8)\nInserts one or more vertices into a graph space in NebulaGraph.\nDELETE VERTEX\nDELETE VERTEX [, ...]\nDELETE VERTEX \"team1\"\nDeletes vertices and the related incoming and outgoing edges of the vertices.\nUPDATE VERTEX\nUPDATE VERTEX ON SET [WHEN ] [YIELD ]\nUPDATE VERTEX ON player \"player101\" SET age = age + 2\nUpdates properties on tags of a vertex.\nUPSERT VERTEX\nUPSERT VERTEX ON SET [WHEN ] [YIELD ]\nUPSERT VERTEX ON player \"player667\" SET age = 31\nThe UPSERT statement is a combination of UPDATE and INSERT. You can use UPSERT VERTEX to update the properties of a vertex if it exists or insert a new vertex if it does not exist.\nEdge statements\nStatement\nSyntax\nExample\nDescription\nINSERT EDGE\nINSERT EDGE [IF NOT EXISTS] ( ) VALUES -> [@] : ( ) [, -> [@] : ( ), ...]\nINSERT EDGE e2 (name, age) VALUES \"11\"->\"13\":(\"n1\", 1)\nInserts an edge or multiple edges into a graph space from a source vertex (given by src_vid) to a destination vertex (given by dst_vid) with a specific rank in NebulaGraph.\nDELETE EDGE\nDELETE EDGE -> [@] [, -> [@] ...]\nDELETE EDGE serve \"player100\" -> \"team204\"@0\nDeletes one edge or multiple edges at a time.\nUPDATE EDGE\nUPDATE EDGE ON -> [@] SET [WHEN ] [YIELD ]\nUPDATE EDGE ON serve \"player100\" -> \"team204\"@0 SET start_year = start_year + 1\nUpdates properties on an edge.\nUPSERT EDGE\nUPSERT EDGE ON -> [@rank] SET [WHEN ] [YIELD ]\nUPSERT EDGE on serve \"player666\" -> \"team200\"@0 SET end_year = 2021\nThe UPSERT statement is a combination of UPDATE and INSERT. You can use UPSERT EDGE to update the properties of an edge if it exists or insert a new edge if it does not exist.\nIndex\nNative index\nYou can use native indexes together with LOOKUP and MATCH statements.\nStatement\nSyntax\nExample\nDescription\nCREATE INDEX\nCREATE {TAG | EDGE} INDEX [IF NOT EXISTS] ON { | } ([]) [COMMENT = '']\nCREATE TAG INDEX player_index on player()\nAdd native indexes for the existing tags, edge types, or properties.\nSHOW CREATE INDEX\nSHOW CREATE {TAG | EDGE} INDEX \nshow create tag index index_2\nShows the statement used when creating a tag or an edge type. It contains detailed information about the index, such as its associated properties.\nSHOW INDEXES\nSHOW {TAG | EDGE} INDEXES\nSHOW TAG INDEXES\nShows the defined tag or edge type indexes names in the current graph space.\nDESCRIBE INDEX\nDESCRIBE {TAG | EDGE} INDEX \nDESCRIBE TAG INDEX player_index_0\nGets the information about the index with a given name, including the property name (Field) and the property type (Type) of the index.\nREBUILD INDEX\nREBUILD {TAG | EDGE} INDEX []\nREBUILD TAG INDEX single_person_index\nRebuilds the created tag or edge type index. If data is updated or inserted before the creation of the index, you must rebuild the indexes manually to make sure that the indexes contain the previously added data.\nSHOW INDEX STATUS\nSHOW {TAG | EDGE} INDEX STATUS\nSHOW TAG INDEX STATUS\nReturns the name of the created tag or edge type index and its status.\nDROP INDEX\nDROP {TAG | EDGE} INDEX [IF EXISTS] \nDROP TAG INDEX player_index_0\nRemoves an existing index from the current graph space.\nFull-text index\nSyntax\nExample\nDescription\nSIGN IN TEXT SERVICE [( [,, ]), (), ...]\nSIGN IN TEXT SERVICE (127.0.0.1:9200)\nThe full-text indexes is implemented based on Elasticsearch. After deploying an Elasticsearch cluster, you can use the SIGN IN statement to log in to the Elasticsearch client.\nSHOW TEXT SEARCH CLIENTS\nSHOW TEXT SEARCH CLIENTS\nShows text search clients.\nSIGN OUT TEXT SERVICE\nSIGN OUT TEXT SERVICE\nSigns out to the text search clients.\nCREATE FULLTEXT {TAG | EDGE} INDEX ON { | } ( [,]...) [ANALYZER=\"\"]\nCREATE FULLTEXT TAG INDEX nebula_index_1 ON player(name)\nCreates full-text indexes.\nSHOW FULLTEXT INDEXES\nSHOW FULLTEXT INDEXES\nShow full-text indexes.\nREBUILD FULLTEXT INDEX\nREBUILD FULLTEXT INDEX\nRebuild full-text indexes.\nDROP FULLTEXT INDEX \nDROP FULLTEXT INDEX nebula_index_1\nDrop full-text indexes.\nLOOKUP ON { | } WHERE ES_QUERY(, \"\") YIELD [| LIMIT [,] ]\nLOOKUP ON player WHERE ES_QUERY(fulltext_index_1,\"Chris\") YIELD id(vertex)\nUse query options.\nSubgraph and path statements\nType\nSyntax\nExample\nDescription\nGET SUBGRAPH\nGET SUBGRAPH [WITH PROP] [ {STEP|STEPS}] FROM {, ...} [{IN | OUT | BOTH} , ...] YIELD [VERTICES AS ] [,EDGES AS ]\nGET SUBGRAPH 1 STEPS FROM \"player100\" YIELD VERTICES AS nodes, EDGES AS relationships\nRetrieves information of vertices and edges reachable from the source vertices of the specified edge types and returns information of the subgraph.\nFIND PATH\nFIND { SHORTEST | ALL | NOLOOP } PATH [WITH PROP] FROM TO OVER [REVERSELY | BIDIRECT] [] [UPTO {STEP|STEPS}] YIELD path as [| ORDER BY $-.path] [| LIMIT ]\nFIND SHORTEST PATH FROM \"player102\" TO \"team204\" OVER * YIELD path as p\nFinds the paths between the selected source vertices and destination vertices. A returned path is like ()-[:@]->(\nEXPLAIN format=\"row\" SHOW TAGSEXPLAIN format=\"dot\" SHOW TAGS\nHelps output the execution plan of an nGQL statement without executing the statement.\nPROFILE\nPROFILE [format=\"row\" | \"dot\"] \nPROFILE format=\"row\" SHOW TAGSEXPLAIN format=\"dot\" SHOW TAGS\nExecutes the statement, then outputs the execution plan as well as the execution profile.\nOperation and maintenance statements\nSUBMIT JOB BALANCE\nSyntax\nDescription\nBALANCE LEADER\nStarts a job to balance the distribution of all the storage leaders in graph spaces. It returns the job ID.\nJob statements\nSyntax\nDescription\nSUBMIT JOB COMPACT\nTriggers the long-term RocksDB compact operation.\nSUBMIT JOB FLUSH\nWrites the RocksDB memfile in the memory to the hard disk.\nSUBMIT JOB STATS\nStarts a job that makes the statistics of the current graph space. Once this job succeeds, you can use the SHOW STATS statement to list the statistics.\nSHOW JOB \nShows the information about a specific job and all its tasks in the current graph space. The Meta Service parses a SUBMIT JOB request into multiple tasks and assigns them to the nebula-storaged processes.\nSHOW JOBS\nLists all the unexpired jobs in the current graph space.\nSTOP JOB\nStops jobs that are not finished in the current graph space.\nRECOVER JOB\nRe-executes the failed jobs in the current graph space and returns the number of recovered jobs.\nKill queries\nSyntax\nExample\nDescription\nKILL QUERY (session=, plan=)\nKILL QUERY(SESSION=1625553545984255,PLAN=163)\nTerminates the query being executed, and is often used to terminate slow queries.\nKill sessions\n | Syntax | Example | Description |\n | ------------------------------------------------------ | --------------------------------------------------- | --------------------------------------------------- |\n | KILL {SESSION|SESSIONS} | KILL SESSION 1672887983842984 | Terminates a single session. |\n | SHOW SESSIONS | YIELD $-.SessionId AS sid [WHERE ] | KILL {SESSION|SESSIONS} $-.sid | SHOW SESSIONS | YIELD $-.SessionId AS sid, $-.CreateTime as CreateTime | ORDER BY $-.CreateTime ASC | LIMIT 2 | KILL SESSIONS $-.sid | Terminates multiple sessions based on specified criteria. |\n | SHOW SESSIONS | KILL SESSIONS $-.SessionId | SHOW SESSIONS | KILL SESSIONS $-.SessionId | Terminates all sessions. |","url":"https://docs.nebula-graph.io/3.6.0/2.quick-start/6.cheatsheet-for-ngql/","type":"doc"},{"title":"Numeric types","content":"nGQL supports both integer and floating-point number.\nInteger\nSigned 64-bit integer (INT64), 32-bit integer (INT32), 16-bit integer (INT16), and 8-bit integer (INT8) are supported.\nType\nDeclared keywords\nRange\nINT64\nINT64 orINT\n-9,223,372,036,854,775,808 ~ 9,223,372,036,854,775,807\nINT32\nINT32\n-2,147,483,648 ~ 2,147,483,647\nINT16\nINT16\n-32,768 ~ 32,767\nINT8\nINT8\n-128 ~ 127\nFloating-point number\nBoth single-precision floating-point format (FLOAT) and double-precision floating-point format (DOUBLE) are supported.\nType\nDeclared keywords\nRange\nPrecision\nFLOAT\nFLOAT\n3.4E +/- 38\n6~7 bits\nDOUBLE\nDOUBLE\n1.7E +/- 308\n15~16 bits\nScientific notation is also supported, such as 1e2, 1.1e2, .3e4, 1.e4, and -1234E-10.\nReading and writing of data values\nWhen writing and reading different types of data, nGQL complies with the following rules:\nData type\nSet as VID\nSet as property\nResulted data type\nINT64\nSupported\nSupported\nINT64\nINT32\nNot supported\nSupported\nINT64\nINT16\nNot supported\nSupported\nINT64\nINT8\nNot supported\nSupported\nINT64\nFLOAT\nNot supported\nSupported\nDOUBLE\nDOUBLE\nNot supported\nSupported\nDOUBLE\nFor example, nGQL does not support setting VID as INT8, but supports setting a certain property type of TAG or Edge type as INT8. When using the nGQL statement to read the property of INT8, the resulted type is INT64.\nMultiple formats are supported:\nDecimal, such as 123456.\nHexadecimal, such as 0x1e240.\nOctal, such as 0361100.\nHowever, NebulaGraph will parse the written non-decimal value into a decimal value and save it. The value read is decimal.\nFor example, the type of the property score is INT. The value of 0xb is assigned to it through the INSERT statement. If querying the property value with statements such as FETCH, you will get the result 11, which is the decimal result of the hexadecimal 0xb.\nRound a FLOAT/DOUBLE value when inserting it to an INT column.","url":"3.data-types/1.numeric/","type":"doc"},{"title":"Boolean","content":"A boolean data type is declared with the bool keyword and can only take the values true or false.\nnGQL supports using boolean in the following ways:\nDefine the data type of the property value as a boolean.\nUse boolean as judgment conditions in the WHERE clause.","url":"3.data-types/2.boolean/","type":"doc"},{"title":"String","content":"Fixed-length strings and variable-length strings are supported.\nDeclaration and literal representation\nThe string type is declared with the keywords of:\nSTRING: Variable-length strings.\nFIXED_STRING(): Fixed-length strings. is the length of the string, such as FIXED_STRING(32).\nA string type is used to store a sequence of characters (text). The literal constant is a sequence of characters of any length surrounded by double or single quotes. For example, \"Hello, Cooper\" or 'Hello, Cooper'.\nString reading and writing\nNebula Graph supports using string types in the following ways:\nDefine the data type of VID as a fixed-length string.\nSet the variable-length string as the Schema name, including the names of the graph space, tag, edge type, and property.\nDefine the data type of the property as a fixed-length or variable-length string.\nFor example:\nDefine the data type of the property as a fixed-length string CREATE TAG IF NOT EXISTS t1 (p1 FIXED_STRING(10)); \nDefine the data type of the property as a variable-length string CREATE TAG IF NOT EXISTS t2 (p2 STRING); \nWhen the fixed-length string you try to write exceeds the length limit:\nIf the fixed-length string is a property, the writing will succeed, and NebulaGraph will truncate the string and only store the part that meets the length limit.\nIf the fixed-length string is a VID, the writing will fail and NebulaGraph will return an error.\nEscape characters\nLine breaks are not allowed in a string. Escape characters are supported within strings, for example:\n\"\\n\\t\\r\\b\\f\"\n\"\\110ello world\"\nOpenCypher compatibility\nThere are some tiny differences between openCypher and Cypher, as well as nGQL. The following is what openCypher requires. Single quotes cannot be converted to double quotes.\n# File: Literals.feature\nFeature: Literals\nBackground:\n Given any graph\n Scenario: Return a single-quoted string\n When executing query:\n \"\"\"\n RETURN '' AS literal\n \"\"\"\n Then the result should be, in any order:\n | literal |\n | '' | # Note: it should return single-quotes as openCypher required.\n And no side effects\nWhile Cypher accepts both single quotes and double quotes as the return results. nGQL follows the Cypher way.\nnebula > YIELD '' AS quote1, \"\" AS quote2, \"'\" AS quote3, '\"' AS quote4","url":"3.data-types/3.string/","type":"doc"},{"title":"Date and time types","content":"This topic will describe the DATE, TIME, DATETIME, TIMESTAMP, and DURATION types.\nPrecautions\nWhile inserting time-type property values with DATE, TIME, and DATETIME, NebulaGraph transforms them to a UTC time according to the timezone specified with the timezone_name parameter in the configuration files. \ndate(), time(), and datetime() can convert a time-type property with a specified timezone. For example, datetime(\"2017-03-04 22:30:40.003000+08:00\") or datetime(\"2017-03-04T22:30:40.003000[Asia/Shanghai]\").\ndate(), time(), datetime(), and timestamp() all accept empty parameters to return the current date, time, and datetime.\ndate(), time(), and datetime() all accept the property name to return a specific property value of itself. For example, date().month returns the current month, while time(\"02:59:40\").minute returns the minutes of the importing time.\nFor time operations it is recommended to use duration() to calculate the offset of the moment. Addition and subtraction of date() and date(), timestamp() and timestamp() are also supported.\nWhen setting the year of the time as a negative number, you need to use Map type data.\nOpenCypher Compatibility\nIn nGQL:\nYear, month, day, hour, minute, second, millisecond, and microsecond are supported, while the nanosecond is not supported.\nlocaldatetime() is not supported.\nMost string time formats are not supported. The exceptions are YYYY-MM-DDThh:mm:ss and YYYY-MM-DD hh:mm:ss.\nThe single-digit string time format is supported. For example, time(\"1:1:1\").\nDATE\nThe DATE type is used for values with a date part but no time part. Nebula Graph retrieves and displays DATE values in the YYYY-MM-DD format. The supported range is -32768-01-01 to 32767-12-31.\nThe properties of date() include year, month, and day. date() supports the input of YYYYY, YYYYY-MM or YYYYY-MM-DD, and defaults to 01 for an untyped month or day.\n RETURN DATE({year:-123, month:12, day:3});\n RETURN DATE(\"23333\");\n RETURN DATE(\"2023-12-12\") - DATE(\"2023-12-11\");\nTIME\nThe TIME type is used for values with a time part but no date part. Nebula Graph retrieves and displays TIME values in hh:mm:ss.msmsmsususus format. The supported range is 00:00:00.000000 to 23:59:59.999999.\nThe properties of time() include hour, minute, and second.\nDATETIME\nThe DATETIME type is used for values that contain both date and time parts. Nebula Graph retrieves and displays DATETIME values in YYYY-MM-DDThh:mm:ss.msmsmsususus format. The supported range is -32768-01-01T00:00:00.000000 to 32767-12-31T23:59:59.999999.\nThe properties of datetime() include year, month, day, hour, minute, and second.\ndatetime() can convert TIMESTAMP to DATETIME. The value range of TIMESTAMP is 0~9223372036.\ndatetime() supports an int argument. The int argument specifies a timestamp.\n# To get the current date and time.\n RETURN datetime();\n# To get the current hour.\n RETURN datetime().hour;\n# To get date time from a given timestamp.\n RETURN datetime(timestamp(1625469277));\n RETURN datetime(1625469277);\nTIMESTAMP\nThe TIMESTAMP data type is used for values that contain both date and time parts. It has a range of 1970-01-01T00:00:01 UTC to 2262-04-11T23:47:16 UTC.\nTIMESTAMP has the following features:\nStored and displayed in the form of a timestamp, such as 1615974839, which means 2021-03-17T17:53:59.\nSupported TIMESTAMP querying methods: timestamp and timestamp() function.\nSupported TIMESTAMP inserting methods: timestamp, timestamp() function, and now() function.\ntimestamp() function accepts empty arguments to get the current timestamp. It can pass an integer arguments to identify the integer as a timestamp and the range of passed integer is: 0~9223372036。\ntimestamp() function can convert DATETIME to TIMESTAMP, and the data type of DATETIME should be a string. \nThe underlying storage data type is int64.\n# To get the current timestamp.\n RETURN timestamp();\n# To get a timestamp from given date and time.\n RETURN timestamp(\"2022-01-05T06:18:43\");\n# To get a timestamp using datetime().\n RETURN timestamp(datetime(\"2022-08-29T07:53:10.939000\"));\n+---------------------------------------------------+ \nDURATION\nThe DURATION data type is used to indicate a period of time. Map data that are freely combined by years, months, days, hours, minutes, and seconds indicates the DURATION.\nDURATION has the following features:\nCreating indexes for DURATION is not supported.\nDURATION can be used to calculate the specified time.\nExamples\nCreate a tag named date1 with three properties: DATE, TIME, and DATETIME.\n CREATE TAG IF NOT EXISTS date1(p1 date, p2 time, p3 datetime);\nInsert a vertex named test1.\n INSERT VERTEX date1(p1, p2, p3) VALUES \"test1\":(date(\"2021-03-17\"), time(\"17:53:59\"), datetime(\"2017-03-04T22:30:40.003000[Asia/Shanghai]\"));\nQuery whether the value of property p1 on the test1 tag is 2021-03-17.\n MATCH (v:date1) RETURN v.date1.p1 == date(\"2021-03-17\");\nReturn the content of the property p1 on test1.\n CREATE TAG INDEX IF NOT EXISTS date1_index ON date1(p1);\n REBUILD TAG INDEX date1_index;\n MATCH (v:date1) RETURN v.date1.p1;\nSearch for vertices with p3 property values less than 2023-01-01T00:00:00.000000, and return the p3 values.\n MATCH (v:date1) \\\nWHERE v.date1.p3 < datetime(\"2023-01-01T00:00:00.000000\") \\\nRETURN v.date1.p3;\nCreate a tag named school with the property of TIMESTAMP.\n CREATE TAG IF NOT EXISTS school(name string , found_time timestamp);\nInsert a vertex named DUT with a found-time timestamp of \"1988-03-01T08:00:00\".\n# Insert as a timestamp. The corresponding timestamp of 1988-03-01T08:00:00 is 573177600, or 573206400 UTC.\n INSERT VERTEX school(name, found_time) VALUES \"DUT\":(\"DUT\", 573206400);\n# Insert in the form of date and time.\n INSERT VERTEX school(name, found_time) VALUES \"DUT\":(\"DUT\", timestamp(\"1988-03-01T08:00:00\"));\nInsert a vertex named dut and store time with now() or timestamp() functions.\n# Use now() function to store time\n INSERT VERTEX school(name, found_time) VALUES \"dut\":(\"dut\", now());\n# Use timestamp() function to store time\n INSERT VERTEX school(name, found_time) VALUES \"dut\":(\"dut\", timestamp());\nYou can also use WITH statement to set a specific date and time, or to perform calculations. For example:\n WITH time({hour: 12, minute: 31, second: 14, millisecond:111, microsecond: 222}) AS d RETURN d;\n WITH date({year: 1984, month: 10, day: 11}) AS x RETURN x + 1;\n WITH date('1984-10-11') as x, duration({years: 12, days: 14, hours: 99, minutes: 12}) as d \\\n RETURN x + d AS sum, x - d AS diff;","url":"3.data-types/4.date-and-time/","type":"doc"},{"title":"NULL","content":"You can set the properties for vertices or edges to NULL. Also, you can set the NOT NULL constraint to make sure that the property values are NOT NULL. If not specified, the property is set to NULL by default.\nLogical operations with NULL\nHere is the truth table for AND, OR, XOR, and NOT.\na AND b\na OR b\na XOR b\nNOT a\nfalse\nfalse\nfalse\nfalse\nfalse\ntrue\nfalse\nnull\nfalse\nnull\nnull\ntrue\nfalse\ntrue\nfalse\ntrue\ntrue\ntrue\ntrue\nfalse\nfalse\ntrue\ntrue\nfalse\ntrue\nnull\nnull\ntrue\nnull\nfalse\ntrue\ntrue\ntrue\ntrue\nfalse\nfalse\nnull\nfalse\nfalse\nnull\nnull\nnull\nnull\nnull\nnull\nnull\nnull\nnull\nnull\ntrue\nnull\ntrue\nnull\nnull\nOpenCypher compatibility\nThe comparisons and operations about NULL are different from openCypher. There may be changes later.\nComparisons with NULL\nThe comparison operations with NULL are incompatible with openCypher.\nOperations and RETURN with NULL\nThe NULL operations and RETURN with NULL are incompatible with openCypher.\nExamples\nUse NOT NULL\nCreate a tag named player. Specify the property name as NOT NULL.\n CREATE TAG IF NOT EXISTS player(name string NOT NULL, age int);\nUse SHOW to create tag statements. The property name is NOT NULL. The property age is NULL by default.\n SHOW CREATE TAG player;\nInsert the vertex Kobe. The property age can be NULL.\n INSERT VERTEX player(name, age) VALUES \"Kobe\":(\"Kobe\",null);\nUse NOT NULL and set the default\nCreate a tag named player. Specify the property age as NOT NULL. The default value is 18.\n CREATE TAG IF NOT EXISTS player(name string, age int NOT NULL DEFAULT 18);\nInsert the vertex Kobe. Specify the property name only.\n INSERT VERTEX player(name) VALUES \"Kobe\":(\"Kobe\");\nQuery the vertex Kobe. The property age is 18 by default.\n FETCH PROP ON player \"Kobe\" YIELD properties(vertex);","url":"3.data-types/5.null/","type":"doc"},{"title":"Lists","content":"The list is a composite data type. A list is a sequence of values. Individual elements in a list can be accessed by their positions.\nA list starts with a left square bracket [ and ends with a right square bracket ]. A list contains zero, one, or more expressions. List elements are separated from each other with commas (,). Whitespace around elements is ignored in the list, thus line breaks, tab stops, and blanks can be used for formatting.\nOpenCypher compatibility\nA composite data type (i.e. set, map, and list) CANNOT be stored as properties of vertices or edges.\nList operations\nYou can use the preset list function to operate the list, or use the index to filter the elements in the list.\nIndex syntax\n[M]\n[M..N]\n[M..]\n[..N]\nThe index of nGQL supports queries from front to back, starting from 0. 0 means the first element, 1 means the second element, and so on. It also supports queries from back to front, starting from -1. -1 means the last element, -2 means the penultimate element, and so on.\n[M]: represents the element whose index is M.\n[M..N]: represents the elements whose indexes are greater or equal to M but smaller than N. Return empty when N is 0.\n[M..]: represents the elements whose indexes are greater or equal to M.\n[..N]: represents the elements whose indexes are smaller than N. Return empty when N is 0.\nExamples\n# The following query returns the list [1,2,3].\n RETURN list[1, 2, 3] AS a;\n# The following query returns the element whose index is 3 in the list [1,2,3,4,5]. In a list, the index starts from 0, and thus the return element is 4.\n RETURN range(1,5)[3];\n# The following query returns the element whose index is -2 in the list [1,2,3,4,5]. The index of the last element in a list is -1, and thus the return element is 4.\n RETURN range(1,5)[-2];\n# The following query returns the elements whose indexes are from 0 to 3 (not including 3) in the list [1,2,3,4,5].\n RETURN range(1,5)[0..3];\n# The following query returns the elements whose indexes are greater than 2 in the list [1,2,3,4,5].\n RETURN range(1,5)[3..] AS a;\n# The following query returns the elements whose indexes are smaller than 3.\n WITH list[1, 2, 3, 4, 5] AS a \\\n RETURN a[..3] AS r;\n# The following query filters the elements whose indexes are greater than 2 in the list [1,2,3,4,5], calculate them respectively, and returns them.\n RETURN [n IN range(1,5) WHERE n > 2 | n + 10] AS a;\n# The following query returns the elements from the first to the penultimate (inclusive) in the list [1, 2, 3].\n YIELD list[1, 2, 3][0..-1] AS a;\n# The following query returns the elements from the first (exclusive) to the third backward in the list [1, 2, 3, 4, 5].\n YIELD list[1, 2, 3, 4, 5][-3..-1] AS a;\n# The following query sets the variables and returns the elements whose indexes are 1 and 2.\n $var = YIELD 1 AS f, 3 AS t; \\\n YIELD list[1, 2, 3][$var.f..$var.t] AS a;\n# The following query returns empty because the index is out of bound. It will return normally when the index is within the bound.\n RETURN list[1, 2, 3, 4, 5] [0..10] AS a;\n RETURN list[1, 2, 3] [-5..5] AS a;\n# The following query returns empty because there is a [0..0].\n RETURN list[1, 2, 3, 4, 5] [0..0] AS a;\n# The following query returns empty because of M ≥ N.\n RETURN list[1, 2, 3, 4, 5] [3..1] AS a;\n# When conduct a range query, if `M` or `N` is null, return `null`.\n WITH list[1,2,3] AS a \\\n RETURN a[0..null] as r;\n# The following query calculates the elements in the list [1,2,3,4,5] respectively and returns them without the list head.\n RETURN tail([n IN range(1, 5) | 2 * n - 10]) AS a;\n# The following query takes the elements in the list [1,2,3] as true and return.\n RETURN [n IN range(1, 3) WHERE true | n] AS r;\n# The following query returns the length of the list [1,2,3].\n RETURN size(list[1,2,3]);\n# The following query calculates the elements in the list [92,90] and runs a conditional judgment in a where clause.\n GO FROM \"player100\" OVER follow WHERE properties(edge).degree NOT IN [x IN [92, 90] | x + $$.player.age] \\\n YIELD dst(edge) AS id, properties(edge).degree AS degree;\n# The following query takes the query result of the MATCH statement as the elements in a list. Then it calculates and returns them.\n MATCH p = (n:player{name:\"Tim Duncan\"})-[:follow]->(m) \\\n RETURN [n IN nodes(p) | n.player.age + 100] AS r;\nOpenCypher compatibility\nIn openCypher, return null when querying a single out-of-bound element. However, in nGQL, return OUT_OF_RANGE when querying a single out-of-bound element. RETURN range(0,5)[-12];\nA composite data type (i.e., set, map, and list) CAN NOT be stored as properties for vertices or edges.\nIt is recommended to modify the graph modeling method. The composite data type should be modeled as an adjacent edge of a vertex, rather than its property. Each adjacent edge can be dynamically added or deleted. The rank values of the adjacent edges can be used for sequencing.\nPatterns are not supported in the list. For example, [(src)-[]->(m) | m.name].","url":"3.data-types/6.list/","type":"doc"},{"title":"Sets","content":"The set is a composite data type. A set is a set of values. Unlike a List, values in a set are unordered and each value must be unique.\nA set starts with a left curly bracket { and ends with a right curly bracket }. A set contains zero, one, or more expressions. Set elements are separated from each other with commas (,). Whitespace around elements is ignored in the set, thus line breaks, tab stops, and blanks can be used for formatting.\nOpenCypher compatibility\nA composite data type (i.e. set, map, and list) CANNOT be stored as properties of vertices or edges.\nA set is not a data type in openCypher, but in nGQL, users can use the set.\nExamples\n# The following query returns the set {1,2,3}.\n RETURN set{1, 2, 3} AS a;\n# The following query returns the set {1,2}, Because the set does not allow repeating elements, and the order is unordered.\n RETURN set{1, 2, 1} AS a;\n# The following query checks whether the set has the specified element 1.\n RETURN 1 IN set{1, 2} AS a;\n# The following query counts the number of elements in the set.\n YIELD size(set{1, 2, 1}) AS a;\n# The following query returns a set of target vertex property values.\n GO FROM \"player100\" OVER follow \\\n YIELD set{properties($$).name,properties($$).age} as a;","url":"3.data-types/7.set/","type":"doc"},{"title":"Maps","content":"The map is a composite data type. Maps are unordered collections of key-value pairs. In maps, the key is a string. The value can have any data type. You can get the map element by using map['key'].\nA map starts with a left curly bracket { and ends with a right curly bracket }. A map contains zero, one, or more key-value pairs. Map elements are separated from each other with commas (,). Whitespace around elements is ignored in the map, thus line breaks, tab stops, and blanks can be used for formatting.\nOpenCypher compatibility\nA composite data type (i.e. set, map, and list) CANNOT be stored as properties of vertices or edges.\nMap projection is not supported.\nExamples\n# The following query returns the simple map.\n YIELD map{key1: 'Value1', Key2: 'Value2'} as a;\n# The following query returns the list type map.\n YIELD map{listKey: [{inner: 'Map1'}, {inner: 'Map2'}]} as a;\n# The following query returns the hybrid type map.\n RETURN map{a: LIST[1,2], b: SET{1,2,1}, c: \"hee\"} as a;\n# The following query returns the specified element in a map.\n RETURN map{a: LIST[1,2], b: SET{1,2,1}, c: \"hee\"}[\"b\"] AS b;\n# The following query checks whether the map has the specified key, not support checks whether the map has the specified value yet.\n RETURN \"a\" IN MAP{a:1, b:2} AS a;","url":"3.data-types/8.map/","type":"doc"},{"title":"Type Conversion/Type coercions","content":"Converting an expression of a given type to another type is known as type conversion.\nNebulaGraph supports converting expressions explicit to other types. For details, see Type conversion functions.\nExamples\n UNWIND [true, false, 'true', 'false', NULL] AS b \\\n RETURN toBoolean(b) AS b;\n RETURN toFloat(1), toFloat('1.3'), toFloat('1e3'), toFloat('not a number');","url":"3.data-types/9.type-conversion/","type":"doc"},{"title":"Geography","content":"Geography is a data type composed of latitude and longitude that represents geospatial information. NebulaGraph currently supports Point, LineString, and Polygon in Simple Features and some functions in SQL-MM 3, such as part of the core geo parsing, construction, formatting, conversion, predicates, and dimensions.\nType description\nA point is the basic data type of geography, which is determined by a latitude and a longitude. For example, \"POINT(3 8)\" means that the longitude is 3° and the latitude is 8°. Multiple points can form a linestring or a polygon.\nShape\nExample\nDescription\nPoint\n\"POINT(3 8)\"\nSpecifies the data type as a point.\nLineString\n\"LINESTRING(3 8, 4.7 73.23)\"\nSpecifies the data type as a linestring.\nPolygon\n\"POLYGON((0 1, 1 2, 2 3, 0 1))\"\nSpecifies the data type as a polygon.\nExamples\n//Create a Tag to allow storing any geography data type.\n CREATE TAG IF NOT EXISTS any_shape(geo geography);\n//Create a Tag to allow storing a point only.\n CREATE TAG IF NOT EXISTS only_point(geo geography(point));\n//Create a Tag to allow storing a linestring only.\n CREATE TAG IF NOT EXISTS only_linestring(geo geography(linestring));\n//Create a Tag to allow storing a polygon only.\n CREATE TAG IF NOT EXISTS only_polygon(geo geography(polygon));\n//Create an Edge type to allow storing any geography data type.\n CREATE EDGE IF NOT EXISTS any_shape_edge(geo geography);\n//Create a vertex to store the geography of a polygon.\n INSERT VERTEX any_shape(geo) VALUES \"103\":(ST_GeogFromText(\"POLYGON((0 1, 1 2, 2 3, 0 1))\"));\n//Create an edge to store the geography of a polygon.\n INSERT EDGE any_shape_edge(geo) VALUES \"201\"->\"302\":(ST_GeogFromText(\"POLYGON((0 1, 1 2, 2 3, 0 1))\"));\n//Query the geography of Vertex 103.\n FETCH PROP ON any_shape \"103\" YIELD ST_ASText(any_shape.geo);\n//Query the geography of the edge which traverses from Vertex 201 to Vertex 302.\n FETCH PROP ON any_shape_edge \"201\"->\"302\" YIELD ST_ASText(any_shape_edge.geo);\n//Create an index for the geography of the Tag any_shape and run LOOKUP.\n CREATE TAG INDEX IF NOT EXISTS any_shape_geo_index ON any_shape(geo);\n REBUILD TAG INDEX any_shape_geo_index;\n LOOKUP ON any_shape YIELD ST_ASText(any_shape.geo);\nWhen creating an index for geography properties, you can specify the parameters for the index.\nParameter\nDefault value\nDescription\ns2_max_level\n30\nThe maximum level of S2 cell used in the covering. Allowed values: 1~30. Setting it to less than the default means that NebulaGraph will be forced to generate coverings using larger cells.\ns2_max_cells\nThe maximum number of S2 cells used in the covering. Provides a limit on how much work is done exploring the possible coverings. Allowed values: 1~30. You may want to use higher values for odd-shaped regions such as skinny rectangles.\n CREATE TAG INDEX IF NOT EXISTS any_shape_geo_index ON any_shape(geo) with (s2_max_level=30, s2_max_cells=8);\nFor more index information, see Index overview.","url":"3.data-types/10.geography/","type":"doc"},{"title":"Composite queries (clause structure)","content":"Composite queries put data from different queries together. They then use filters, group-bys, or sorting before returning the combined return results.\nNebula Graph supports three methods to run composite queries (or sub-queries):\n(openCypher) Clauses are chained together, and they feed intermediate result sets between each other.\n(Native nGQL) More than one query can be batched together, separated by semicolons (;). The result of the last query is returned as the result of the batch.\n(Native nGQL) Queries can be piped together by using the pipe (|). The result of the previous query can be used as the input of the next query.\nOpenCypher compatibility\nIn a composite query, do not put together openCypher and native nGQL clauses in one statement. For example, this statement is undefined: MATCH ... | GO ... | YIELD ....\nIf you are in the openCypher way (MATCH, RETURN, WITH, etc), do not introduce any pipe or semicolons to combine the sub-clauses.\nIf you are in the native nGQL way (FETCH, GO, LOOKUP, etc), you must use pipe or semicolons to combine the sub-clauses.\nComposite queries are not transactional queries (as in SQL/Cypher)\nFor example, a query is composed of three sub-queries: A B C, A | B | C or A; B; C. In that A is a read operation, B is a computation operation, and C is a write operation. If any part fails in the execution, the whole result will be undefined. There is no rollback. What is written depends on the query executor.\nExamples\nOpenCypher compatibility statement# Connect multiple queries with clauses.\n MATCH p=(v:player{name:\"Tim Duncan\"})--() \\\n WITH nodes(p) AS n \\\n UNWIND n AS n1 \\\n RETURN DISTINCT n1;\nNative nGQL (Semicolon queries)# Only return edges.\n SHOW TAGS; SHOW EDGES;\n# Insert multiple vertices.\n INSERT VERTEX player(name, age) VALUES \"player100\":(\"Tim Duncan\", 42); \\\n INSERT VERTEX player(name, age) VALUES \"player101\":(\"Tony Parker\", 36); \\\n INSERT VERTEX player(name, age) VALUES \"player102\":(\"LaMarcus Aldridge\", 33);\nNative nGQL (Pipe queries)# Connect multiple queries with pipes.\n GO FROM \"player100\" OVER follow YIELD dst(edge) AS id | \\\n GO FROM $-.id OVER serve YIELD properties($$).name AS Team, \\\n properties($^).name AS Player;","url":"4.variable-and-composite-queries/1.composite-queries/","type":"doc"},{"title":"User-defined variables","content":"User-defined variables allow passing the result of one statement to another.\nOpenCypher compatibility\nIn openCypher, when you refer to the vertex, edge, or path of a variable, you need to name it first. For example:\n MATCH (v:player{name:\"Tim Duncan\"}) RETURN v;\nThe user-defined variable in the preceding query is v.\nNative nGQL\nUser-defined variables are written as $var_name. The var_name consists of letters, numbers, or underline characters. Any other characters are not permitted.\nThe user-defined variables are valid only at the current execution (namely, in this composite query). When the execution ends, the user-defined variables will be automatically expired. The user-defined variables in one statement CANNOT be used in any other clients, executions, or sessions.\nYou can use user-defined variables in composite queries. Details about composite queries, see Composite queries.\nExample\n $var = GO FROM \"player100\" OVER follow YIELD dst(edge) AS id; \\\n GO FROM $var.id OVER serve YIELD properties($$).name AS Team, \\\n properties($^).name AS Player;\nSet operations and scope of user-defined variables\nWhen assigning variables within a compound statement involving set operations, it is important to enclose the scope of the variable assignment in parentheses. In the example below, the source of the $var assignment is the results of the output of two INTERSECT statements. \n$var = ( \\\n GO FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS id \\\n INTERSECT \\\n GO FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS id \\\n ); \\\n GO FROM $var.id OVER follow YIELD follow.degree AS degree","url":"4.variable-and-composite-queries/2.user-defined-variables/","type":"doc"},{"title":"Property reference","content":"You can refer to the properties of a vertex or an edge in WHERE and YIELD syntax.\nProperty reference for vertex\nFor source vertex\n$^..\nParameter\nDescription\n$^\nis used to get the property of the source vertex.\ntag_name\nis the tag name of the vertex.\nprop_name\nspecifies the property name.\nFor destination vertex\n$$..\nParameter\nDescription\n$$\nis used to get the property of the destination vertex.\ntag_name\nis the tag name of the vertex.\nprop_name\nspecifies the property name.\nProperty reference for edge\nFor user-defined edge property\n.\nParameter\nDescription\nedge_type\nis the edge type of the edge.\nprop_name\nspecifies the property name of the edge type.\nFor built-in properties\nApart from the user-defined edge property, there are four built-in properties in each edge:\nParameter\nDescription\n_src\nsource vertex ID of the edge\n_dst\ndestination vertex ID of the edge\n_type\nedge type\n_rank\nthe rank value for the edge\nExamples\nThe following query returns the name property of the player tag on the source vertex and the age property of the player tag on the destination vertex.\n GO FROM \"player100\" OVER follow YIELD $^.player.name AS startName, $$.player.age AS endAge;\nThe following query returns the degree property of the edge type follow.\n GO FROM \"player100\" OVER follow YIELD follow.degree;\nThe following query returns the source vertex, the destination vertex, the edge type, and the edge rank value of the edge type follow.\n GO FROM \"player100\" OVER follow YIELD follow._src, follow._dst, follow._type, follow._rank;","url":"4.variable-and-composite-queries/3.property-reference/","type":"doc"},{"title":"Comparison operators","content":"NebulaGraph supports the following comparison operators.\nName\nDescription\n==\nEqual operator\n!=, <>\nNot equal operator\nGreater than operator\n>=\nGreater than or equal operator\nLess than operator\n<=\nLess than or equal operator\nIS NULL\nNULL check\nIS NOT NULL\nNot NULL check\nIS EMPTY\nEMPTY check\nIS NOT EMPTY\nNot EMPTY check\nThe result of the comparison operation is true or false.\nOpenCypher compatibility\nopenCypher does not have EMPTY. Thus EMPTY is not supported in MATCH statements.\nExamples\n==\nString comparisons are case-sensitive. Values of different types are not equal.\n RETURN 'A' == 'a', toUpper('A') == toUpper('a'), toLower('A') == toLower('a');\n RETURN '2' == 2, toInteger('2') == 2;\n RETURN 3 > 2;\n WITH 4 AS one, 3 AS two \\\n RETURN one > two AS result;\n>=\n RETURN 2 >= \"2\", 2 >= 2;\n YIELD 2.0 < 1.9;\n<=\n YIELD 0.11 <= 0.11;\n!=\n YIELD 1 != '1';\nIS [NOT] NULL\n RETURN null IS NULL AS value1, null == null AS value2, null != null AS value3;\n RETURN length(NULL), size(NULL), count(NULL), NULL IS NULL, NULL IS NOT NULL, sin(NULL), NULL + NULL, [1, NULL] IS NULL;\n WITH {name: null} AS `map` \\\n RETURN `map`.name IS NOT NULL;\n WITH {name: 'Mats', name2: 'Pontus'} AS map1, \\\n {name: null} AS map2, {notName: 0, notName2: null } AS map3 \\\n RETURN map1.name IS NULL, map2.name IS NOT NULL, map3.name IS NULL;\n MATCH (n:player) \\\n RETURN n.player.age IS NULL, n.player.name IS NOT NULL, n.player.empty IS NULL;\n...\nIS [NOT] EMPTY\n RETURN null IS EMPTY;\n RETURN \"a\" IS NOT EMPTY;\n GO FROM \"player100\" OVER * WHERE properties($$).name IS NOT EMPTY YIELD dst(edge);","url":"5.operators/1.comparison/","type":"doc"},{"title":"Boolean operators","content":"NebulaGraph supports the following boolean operators.\nName\nDescription\nAND\nLogical AND\nNOT\nLogical NOT\nOR\nLogical OR\nXOR\nLogical XOR\nFor the precedence of the operators, refer to Operator Precedence.\nFor the logical operations with NULL, refer to NULL.\nLegacy version compatibility\nNon-zero numbers cannot be converted to boolean values.","url":"5.operators/2.boolean/","type":"doc"},{"title":"Pipe operators","content":"Multiple queries can be combined using pipe operators in nGQL.\nOpenCypher compatibility\nPipe operators apply to native nGQL only.\nSyntax\nOne major difference between nGQL and SQL is how sub-queries are composed.\nIn SQL, sub-queries are nested in the query statements.\nIn nGQL, the shell style PIPE (|) is introduced into the sub-queries.\nExamples\n GO FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS dstid, properties($$).name AS Name | \\\n GO FROM $-.dstid OVER follow YIELD dst(edge);\nUsers must define aliases in the YIELD clause for the reference operator $- to use, just like $-.dstid in the preceding example.\nPerformance tips\nIn NebulaGraph, pipes will affect the performance. Take A | B as an example, the effects are as follows:\nPipe operators operate synchronously. That is, the data can enter the pipe clause as a whole after the execution of clause A before the pipe operator is completed.\nPipe operators need to be serialized and deserialized, which is executed in a single thread.\nIf A sends a large amount of data to |, the entire query request may be very slow. You can try to split this statement.\nSend A from the application,\nSplit the return results on the application,\nSend to multiple graphd processes concurrently,\nEvery graphd process executes part of B.\nThis is usually much faster than executing a complete A | B with a single graphd process.","url":"5.operators/4.pipe/","type":"doc"},{"title":"Reference operators","content":"NGQL provides reference operators to represent a property in a WHERE or YIELD clause, or the output of the statement before the pipe operator in a composite query.\nOpenCypher compatibility\nReference operators apply to native nGQL only.\nReference operator List\nReference operator\nDescription\n$^\nRefers to a source vertex property. For more information, see Property reference.\n$$\nRefers to a destination vertex property. For more information, see Property reference.\n$-\nRefers to the output of the statement before the pipe operator in a composite query. For more information, see Pipe.\nExamples\n# The following example returns the age of the source vertex and the destination vertex.\n GO FROM \"player100\" OVER follow YIELD properties($^).age AS SrcAge, properties($$).age AS DestAge;\n# The following example returns the name and team of the players that player100 follows.\n GO FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS id | \\\n GO FROM $-.id OVER serve \\\n YIELD $^.player.name AS Player, properties($$).name AS Team;","url":"5.operators/5.property-reference/","type":"doc"},{"title":"Set operators","content":"This topic will describe the set operators, including UNION, UNION ALL, INTERSECT, and MINUS. To combine multiple queries, use these set operators.\nAll set operators have equal precedence. If a nGQL statement contains multiple set operators, NebulaGraph will evaluate them from left to right unless parentheses explicitly specify another order.\nUNION, UNION DISTINCT, and UNION ALL\n UNION [DISTINCT | ALL] [ UNION [DISTINCT | ALL] ...]\nOperator UNION DISTINCT (or by short UNION) returns the union of two sets A and B without duplicated elements.\nOperator UNION ALL returns the union of two sets A and B with duplicated elements.\nThe and must have the same number of columns and data types. Different data types are converted according to the Type Conversion.\nExamples\n# The following statement returns the union of two query results without duplicated elements.\n GO FROM \"player102\" OVER follow YIELD dst(edge) \\\n UNION \\\n GO FROM \"player100\" OVER follow YIELD dst(edge);\n MATCH (v:player) \\\n WITH v.player.name AS v \\\n RETURN n ORDER BY n LIMIT 3 \\\n UNION \\\n UNWIND [\"Tony Parker\", \"Ben Simmons\"] AS n \\\n RETURN n;\n# The following statement returns the union of two query results with duplicated elements.\n GO FROM \"player102\" OVER follow YIELD dst(edge) \\\n UNION ALL \\\n GO FROM \"player100\" OVER follow YIELD dst(edge);\n MATCH (v:player) \\\n WITH v.player.name AS n \\\n RETURN n ORDER BY n LIMIT 3 \\\n UNION ALL \\\n UNWIND [\"Tony Parker\", \"Ben Simmons\"] AS n \\\n RETURN n;\n# UNION can also work with the YIELD statement. The DISTINCT keyword will check duplication by all the columns for every line, and remove duplicated lines if every column is the same.\n GO FROM \"player102\" OVER follow \\\n YIELD dst(edge) AS id, properties(edge).degree AS Degree, properties($$).age AS Age \\\n UNION /* DISTINCT */ \\\n GO FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS id, properties(edge).degree AS Degree, properties($$).age AS Age;\nINTERSECT\n INTERSECT \nOperator INTERSECT returns the intersection of two sets A and B (denoted by A ⋂ B).\nSimilar to UNION, the left and right must have the same number of columns and data types. Different data types are converted according to the Type Conversion.\nExample\n# The following statement returns the intersection of two query results.\n GO FROM \"player102\" OVER follow \\\n YIELD dst(edge) AS id, properties(edge).degree AS Degree, properties($$).age AS Age \\\n INTERSECT \\\n GO FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS id, properties(edge).degree AS Degree, properties($$).age AS Age;\n MATCH (v:player)-[e:follow]->(v2) \\\n WHERE id(v) == \"player102\" \\\n RETURN id(v2) As id, e.degree As Degree, v2.player.age AS Age \\\n INTERSECT \\\n MATCH (v:player)-[e:follow]->(v2) \\\n WHERE id(v) == \"player100\" \\\n RETURN id(v2) As id, e.degree As Degree, v2.player.age AS Age;\n UNWIND [1,2] AS a RETURN a \\\n INTERSECT \\\n UNWIND [1,2,3,4] AS a \\\n RETURN a;\nMINUS\n MINUS \nOperator MINUS returns the subtraction (or difference) of two sets A and B (denoted by A-B). Always pay attention to the order of left and right. The set A-B consists of elements that are in A but not in B.\nExample\n# The following statement returns the elements in the first query result but not in the second query result.\n GO FROM \"player100\" OVER follow YIELD dst(edge) \\\n MINUS \\\n GO FROM \"player102\" OVER follow YIELD dst(edge);\n GO FROM \"player102\" OVER follow YIELD dst(edge) AS id\\\n MINUS \\\n GO FROM \"player100\" OVER follow YIELD dst(edge) AS id;\n MATCH (v:player)-[e:follow]->(v2) \\\n WHERE id(v) ==\"player102\" \\\n RETURN id(v2) AS id\\\n MINUS \\\n MATCH (v:player)-[e:follow]->(v2) \\\n WHERE id(v) ==\"player100\" \\\n RETURN id(v2) AS id;\n UNWIND [1,2,3] AS a RETURN a \\\n MINUS \\\n WITH 4 AS a \\\n RETURN a;\nPrecedence of the set operators and pipe operators\nPlease note that when a query contains a pipe | and a set operator, the pipe takes precedence. Refer to Pipe for details. The query GO FROM 1 UNION GO FROM 2 | GO FROM 3 is the same as the query GO FROM 1 UNION (GO FROM 2 | GO FROM 3).\nExamples\n GO FROM \"player102\" OVER follow \\\n YIELD dst(edge) AS play_dst \\\n UNION \\\n GO FROM \"team200\" OVER serve REVERSELY \\\n YIELD src(edge) AS play_src \\\n | GO FROM $-.play_src OVER follow YIELD dst(edge) AS play_dst;\nThe above query executes the statements in the red bar first and then executes the statement in the green box.\nThe parentheses can change the execution priority. For example:\n (GO FROM \"player102\" OVER follow \\\n YIELD dst(edge) AS play_dst \\\n UNION \\\n GO FROM \"team200\" OVER serve REVERSELY \\\n YIELD src(edge) AS play_dst) \\\n | GO FROM $-.play_dst OVER follow YIELD dst(edge) AS play_dst;\nIn the above query, the statements within the parentheses take precedence. That is, the UNION operation will be executed first, and its output will be executed as the input of the next operation with pipes.","url":"5.operators/6.set/","type":"doc"},{"title":"String operators","content":"You can use the following string operators for concatenating, querying, and matching.\nName\nDescription\nConcatenates strings.\nCONTAINS\nPerforms searchings in strings.\n(NOT) IN\nChecks whether a value is within a set of values.\n(NOT) STARTS WITH\nPerforms matchings at the beginning of a string.\n(NOT) ENDS WITH\nPerforms matchings at the end of a string.\nRegular expressions\nPerform string matchings using regular expressions.\nExamples\n RETURN 'a' + 'b';\n UNWIND 'a' AS a UNWIND 'b' AS b RETURN a + b;\nCONTAINS\nThe CONTAINS operator requires string types on both left and right sides.\n MATCH (s:player)-[e:serve]->(t:team) WHERE id(s) == \"player101\" \\\n AND t.team.name CONTAINS \"ets\" RETURN s.player.name, e.start_year, e.end_year, t.team.name;\n GO FROM \"player101\" OVER serve WHERE (STRING)properties(edge).start_year CONTAINS \"19\" AND \\\n properties($^).name CONTAINS \"ny\" \\\n YIELD properties($^).name, properties(edge).start_year, properties(edge).end_year, properties($$).name;\n GO FROM \"player101\" OVER serve WHERE !(properties($$).name CONTAINS \"ets\") \\\n YIELD properties($^).name, properties(edge).start_year, properties(edge).end_year, properties($$).name;\n(NOT) IN\n RETURN 1 IN [1,2,3], \"Yao\" NOT IN [\"Yi\", \"Tim\", \"Kobe\"], NULL IN [\"Yi\", \"Tim\", \"Kobe\"];\n(NOT) STARTS WITH\n RETURN 'apple' STARTS WITH 'app', 'apple' STARTS WITH 'a', 'apple' STARTS WITH toUpper('a');\n RETURN 'apple' STARTS WITH 'b','apple' NOT STARTS WITH 'app';\n(NOT) ENDS WITH\n RETURN 'apple' ENDS WITH 'app', 'apple' ENDS WITH 'e', 'apple' ENDS WITH 'E', 'apple' ENDS WITH 'b';\nRegular expressions\nNebulaGraph supports filtering by using regular expressions. The regular expression syntax is inherited from std::regex. You can match on regular expressions by using =~ 'regexp'. For example:\n RETURN \"384748.39\" =~ \"\\\\d+(\\\\.\\\\d{2})?\";\n MATCH (v:player) WHERE v.player.name =~ 'Tony.*' RETURN v.player.name;","url":"5.operators/7.string/","type":"doc"},{"title":"List operators","content":"NebulaGraph supports the following list operators:\nList operator\nDescription\nConcatenates lists.\nIN\nChecks if an element exists in a list.\n[]\nAccesses an element(s) in a list using the index operator.\nExamples\n YIELD [1,2,3,4,5]+[6,7] AS myList;\n RETURN size([NULL, 1, 2]);\n RETURN NULL IN [NULL, 1];\n WITH [2, 3, 4, 5] AS numberlist \\\n UNWIND numberlist AS number \\\n WITH number \\\n WHERE number IN [2, 3, 8] \\\n RETURN number;\n WITH ['Anne', 'John', 'Bill', 'Diane', 'Eve'] AS names RETURN names[1] AS result;","url":"5.operators/8.list/","type":"doc"},{"title":"Arithmetic operators","content":"NebulaGraph supports the following arithmetic operators.\nName\nDescription\nAddition operator\nMinus operator\nMultiplication operator\nDivision operator\nModulo operator\nChanges the sign of the argument\nExamples\n RETURN 1+2 AS result;\n RETURN -10+5 AS result;\n RETURN (3*8)%5 AS result;","url":"5.operators/10.arithmetic/","type":"doc"},{"title":"Operator precedence","content":"The following list shows the precedence of nGQL operators in descending order. Operators that are shown together on a line have the same precedence.\n- (negative number)\n!, NOT\n*, /, %\n-, +\n==, >=, >, <=, <, <>, !=\nAND\nOR, XOR\n= (assignment)\nFor operators that occur at the same precedence level within an expression, evaluation proceeds left to right, with the exception that assignments evaluate right to left.\nThe precedence of operators determines the order of evaluation of terms in an expression. To modify this order and group terms explicitly, use parentheses.\nExamples\n RETURN 2+3*5;\n RETURN (2+3)*5;\nOpenCypher compatibility\nIn openCypher, comparisons can be chained arbitrarily, e.g., x < y <= z is equivalent to x < y AND y <= z in openCypher.\nBut in nGQL, x < y <= z is equivalent to (x < y) <= z. The result of (x < y) is a boolean. Compare it with an integer z, and you will get the final result NULL.","url":"5.operators/9.precedence/","type":"doc"},{"title":"Built-in math functions","content":"This topic describes the built-in math functions supported by NebulaGraph.\nabs()\nabs() returns the absolute value of the argument.\nSyntax: abs()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN abs(-10);\n RETURN abs(5-6);\nfloor()\nfloor() returns the largest integer value smaller than or equal to the argument.(Rounds down)\nSyntax: floor()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN floor(9.9);\nceil()\nceil() returns the smallest integer greater than or equal to the argument.(Rounds up)\nSyntax: ceil()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN ceil(9.1);\nround()\nround() returns the rounded value of the specified number. Pay attention to the floating-point precision when using this function.\nSyntax: round(, )\nexpression: An expression of which the result type is double.\ndigit: Decimal digits. If digit is less than 0, round at the left of the decimal point.\nResult type: Double\nExample:\n RETURN round(314.15926, 2);\n RETURN round(314.15926, -1);\nsqrt()\nsqrt() returns the square root of the argument.\nSyntax: sqrt()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN sqrt(9);\ncbrt()\ncbrt() returns the cubic root of the argument.\nSyntax: cbrt()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN cbrt(8);\nhypot()\nhypot() returns the hypotenuse of a right-angled triangle.\nSyntax: hypot(,)\nexpression_x, expression_y: An expression of which the result type is double. They represent the side lengths x and y of a right triangle.\nResult type: Double\nExample:\n RETURN hypot(3,2*2);\npow()\npow() returns the result of xy.\nSyntax: pow(,,)\nexpression_x: An expression of which the result type is double. It represents the base x.\nexpression_y: An expression of which the result type is double. It represents the exponential y.\nResult type: Double\nExample:\n RETURN pow(3,3);\nexp()\nexp() returns the result of ex.\nSyntax: exp()\nexpression: An expression of which the result type is double. It represents the exponential x.\nResult type: Double\nExample:\n RETURN exp(2);\nexp2()\nexp2() returns the result of 2x.\nSyntax: exp2()\nexpression: An expression of which the result type is double. It represents the exponential x.\nResult type: Double\nExample:\n RETURN exp2(3);\nlog()\nlog() returns the base-e logarithm of the argument. (\\(log_{e}{N}\\))\nSyntax: log()\nexpression: An expression of which the result type is double. It represents the antilogarithm N.\nResult type: Double\nExample:\n RETURN log(8);\nlog2()\nlog2() returns the base-2 logarithm of the argument. (\\(log_{2}{N}\\))\nSyntax: log2()\nexpression: An expression of which the result type is double. It represents the antilogarithm N.\nResult type: Double\nExample:\n RETURN log2(8);\nlog10()\nlog10() returns the base-10 logarithm of the argument. (\\(log_{10}{N}\\))\nSyntax: log10()\nexpression: An expression of which the result type is double. It represents the antilogarithm N.\nResult type: Double\nExample:\n RETURN log10(100);\nsin()\nsin() returns the sine of the argument. Users can convert angles to radians using the function radians().\nSyntax: sin()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN sin(3);\nasin()\nasin() returns the inverse sine of the argument. Users can convert angles to radians using the function radians().\nSyntax: asin()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN asin(0.5);\ncos()\ncos() returns the cosine of the argument. Users can convert angles to radians using the function radians().\nSyntax: cos()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN cos(0.5);\nacos()\nacos() returns the inverse cosine of the argument. Users can convert angles to radians using the function radians().\nSyntax: acos()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN acos(0.5);\ntan()\ntan() returns the tangent of the argument. Users can convert angles to radians using the function radians().\nSyntax: tan()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN tan(0.5);\natan()\natan() returns the inverse tangent of the argument. Users can convert angles to radians using the function radians().\nSyntax: atan()\nexpression: An expression of which the result type is double.\nResult type: Double\nExample:\n RETURN atan(0.5);\nrand()\nrand() returns a random floating point number in the range from 0 (inclusive) to 1 (exclusive); i.e.[0,1).\nSyntax: rand()\nResult type: Double\nExample:\n RETURN rand();\nrand32()\nrand32() returns a random 32-bit integer in [min, max).\nSyntax: rand32(,)\nexpression_min: An expression of which the result type is int. It represents the minimum min.\nexpression_max: An expression of which the result type is int. It represents the maximum max.\nResult type: Int\nIf you set only one argument, it is parsed as max and min is 0 by default. If you set no argument, the system returns a random signed 32-bit integer.\nExample:\n RETURN rand32(1,100);\nrand64()\nrand64() returns a random 64-bit integer in [min, max).\nSyntax: rand64(,)\nexpression_min: An expression of which the result type is int. It represents the minimum min.\nexpression_max: An expression of which the result type is int. It represents the maximum max.\nResult type: Int\nIf you set only one argument, it is parsed as max and min is 0 by default. If you set no argument, the system returns a random signed 64-bit integer.\nExample:\n RETURN rand64(1,100);\nbit_and()\nbit_and() returns the result of bitwise AND.\nSyntax: bit_and(,)\nexpression_1, expression_2: An expression of which the result type is int.\nResult type: Int\nExample:\n RETURN bit_and(5,6);\nbit_or()\nbit_or() returns the result of bitwise OR.\nSyntax: bit_or(,)\nexpression_1, expression_2: An expression of which the result type is int.\nResult type: Int\nExample:\n RETURN bit_or(5,6);\nbit_xor()\nbit_xor() returns the result of bitwise XOR.\nSyntax: bit_xor(,)\nexpression_1, expression_2: An expression of which the result type is int.\nResult type: Int\nExample:\n RETURN bit_xor(5,6);\nsize()\nsize() returns the number of elements in a list or a map, or the length of a string.\nSyntax: size({|})\nexpression: An expression for a list or map.\nstring: A specified string.\nResult type: Int\nExample:\n RETURN size([1,2,3,4]);\n RETURN size(\"basketballplayer\") as size;\nrange()\nrange() returns a list of integers from [start,end] in the specified steps.\nSyntax: range(,[,])\nexpression_start: An expression of which the result type is int. It represents the starting value start.\nexpression_end: An expression of which the result type is int. It represents the end value end.\nexpression_step: An expression of which the result type is int. It represents the step size step, step is 1 by default.\nResult type: List\nExample:\n RETURN range(1,3*3,2);\nsign()\nsign() returns the signum of the given number. If the number is 0, the system returns 0. If the number is negative, the system returns -1. If the number is positive, the system returns 1.\nSyntax: sign()\nexpression: An expression of which the result type is double.\nResult type: Int\nExample:\n RETURN sign(10);\ne()\ne() returns the base of the natural logarithm, e (2.718281828459045).\nSyntax: e()\nResult type: Double\nExample:\n RETURN e();\npi()\npi() returns the mathematical constant pi (3.141592653589793).\nSyntax: pi()\nResult type: Double\nExample:\n RETURN pi();\nradians()\nradians() converts angles to radians.\nSyntax: radians()\nResult type: Double\nExample:\n RETURN radians(180);","url":"6.functions-and-expressions/1.math/","type":"doc"},{"title":"Aggregating functions","content":"This topic describes the aggregating functions supported by NebulaGraph.\navg()\navg() returns the average value of the argument.\nSyntax: avg()\nResult type: Double\nExample:\n MATCH (v:player) RETURN avg(v.player.age);\ncount()\ncount() returns the number of records.\n(Native nGQL) You can use count() and GROUP BY together to group and count the number of parameters. Use YIELD to return.\n(OpenCypher style) You can use count() and RETURN. GROUP BY is not necessary.\nSyntax: count({ | *})\ncount(*) returns the number of rows (including NULL).\nResult type: Int\nExample:\n WITH [NULL, 1, 1, 2, 2] As a UNWIND a AS b \\\n RETURN count(b), count(*), count(DISTINCT b);\n# The statement in the following example searches for the people whom `player101` follows and people who follow `player101`, i.e. a bidirectional query.\n# Group and count the number of parameters.\n GO FROM \"player101\" OVER follow BIDIRECT \\\n YIELD properties($$).name AS Name \\\n | GROUP BY $-.Name YIELD $-.Name, count(*);\n# Count the number of parameters.\n MATCH (v1:player)-[:follow]-(v2:player) \\\n WHERE id(v1)== \"player101\" \\\n RETURN v2.player.name AS Name, count(*) as cnt ORDER BY cnt DESC;\nThe preceding example retrieves two columns:\n$-.Name: the names of the people.\ncount(*): how many times the names show up.\nBecause there are no duplicate names in the basketballplayer dataset, the number 2 in the column count(*) shows that the person in that row and player101 have followed each other.\n# a: The statement in the following example retrieves the age distribution of the players in the dataset.\n LOOKUP ON player \\\n YIELD player.age As playerage \\\n | GROUP BY $-.playerage \\\n YIELD $-.playerage as age, count(*) AS number \\\n | ORDER BY $-.number DESC, $-.age DESC;\n...\n# b: The statement in the following example retrieves the age distribution of the players in the dataset.\n MATCH (n:player) \\\n RETURN n.player.age as age, count(*) as number \\\n ORDER BY number DESC, age DESC;\n...\n# The statement in the following example counts the number of edges that Tim Duncan relates.\n MATCH (v:player{name:\"Tim Duncan\"}) -[e]- (v2) \\\n RETURN count(e);\n# The statement in the following example counts the number of edges that Tim Duncan relates and returns two columns (no DISTINCT and DISTINCT) in multi-hop queries.\n MATCH (n:player {name : \"Tim Duncan\"})-[]->(friend:player)-[]->(fof:player) \\\n RETURN count(fof), count(DISTINCT fof);\nmax()\nmax() returns the maximum value.\nSyntax: max()\nResult type: Same as the original argument.\nExample:\n MATCH (v:player) RETURN max(v.player.age);\nmin()\nmin() returns the minimum value.\nSyntax: min()\nResult type: Same as the original argument.\nExample:\n MATCH (v:player) RETURN min(v.player.age);\ncollect()\ncollect() returns a list containing the values returned by an expression. Using this function aggregates data by merging multiple records or values into a single list.\nSyntax: collect()\nResult type: List\nExample:\n UNWIND [1, 2, 1] AS a \\\n RETURN a;\n UNWIND [1, 2, 1] AS a \\\n RETURN collect(a);\n UNWIND [1, 2, 1] AS a \\\n RETURN a, collect(a), size(collect(a));\n# The following examples sort the results in descending order, limit output rows to 3, and collect the output into a list.\n UNWIND [\"c\", \"b\", \"a\", \"d\" ] AS p \\\n WITH p AS q \\\n ORDER BY q DESC LIMIT 3 \\\n RETURN collect(q);\n WITH [1, 1, 2, 2] AS coll \\\n UNWIND coll AS x \\\n WITH DISTINCT x \\\n RETURN collect(x) AS ss;\n MATCH (n:player) \\\n RETURN collect(n.player.age);\n| [32, 32, 34, 29, 41, 40, 33, 25, 40, 37, ...\n...\n# The following example aggregates all the players' names by their ages.\n MATCH (n:player) \\\n RETURN n.player.age AS age, collect(n.player.name);\n...\n GO FROM \"player100\" OVER serve \\\n YIELD properties($$).name AS name \\\n | GROUP BY $-.name \\\n YIELD collect($-.name) AS name;\n LOOKUP ON player \\\n YIELD player.age As playerage \\\n | GROUP BY $-.playerage \\\n YIELD collect($-.playerage) AS playerage;\n...\nstd()\nstd() returns the population standard deviation.\nSyntax: std()\nResult type: Double\nExample:\n MATCH (v:player) RETURN std(v.player.age);\nsum()\nsum() returns the sum value.\nSyntax: sum()\nResult type: Same as the original argument.\nExample:\n MATCH (v:player) RETURN sum(v.player.age);\nAggregating example\n GO FROM \"player100\" OVER follow YIELD dst(edge) AS dst, properties($$).age AS age \\\n | GROUP BY $-.dst \\\n YIELD \\\n $-.dst AS dst, \\\n toInteger((sum($-.age)/count($-.age)))+avg(distinct $-.age+1)+1 AS statistics;","url":"6.functions-and-expressions/15.aggregating/","type":"doc"},{"title":"Built-in string functions","content":"This topic describes the built-in string functions supported by NebulaGraph.\nPrecautions\nA string type is used to store a sequence of characters (text). The literal constant is a sequence of characters of any length surrounded by double or single quotes.\nLike SQL, the position index of nGQL starts from 1, while in C language it starts from 0.\nstrcasecmp()\nstrcasecmp() compares string a and b without case sensitivity.\nSyntax: strcasecmp(,)\nstring_a, string_b: Strings to compare.\nResult type: Int\nWhen string_a = string_b, the return value is 0. When string_a > string_b, the return value is greater than 0. When string_a < string_b, the return value is less than 0.\nExample:\n RETURN strcasecmp(\"a\",\"aa\");\nlower() and toLower()\nlower() and toLower() can both returns the argument in lowercase.\nSyntax: lower(), toLower()\nstring: A specified string.\nResult type: String\nExample:\n RETURN lower(\"Basketball_Player\");\nupper() and toUpper()\nupper() and toUpper() can both returns the argument in uppercase.\nSyntax: upper(), toUpper()\nstring: A specified string.\nResult type: String\nExample:\n RETURN upper(\"Basketball_Player\");\nlength()\nlength() returns the length of the given string in bytes.\nSyntax: length({|})\nstring: A specified string.\npath: A specified path represented by a variable.\nResult type: Int\nExample:\n RETURN length(\"basketball\");\n MATCH p=(v:player{name:\"Tim Duncan\"})-->(v2) return length(p);\ntrim()\ntrim() removes the spaces at the leading and trailing of the string.\nSyntax: trim()\nstring: A specified string.\nResult type: String\nExample:\n RETURN trim(\" basketball player \");\nltrim()\nltrim() removes the spaces at the leading of the string.\nSyntax: ltrim()\nstring: A specified string.\nResult type: String\nExample:\n RETURN ltrim(\" basketball player \");\nrtrim()\nrtrim() removes the spaces at the trailing of the string.\nSyntax: rtrim()\nstring: A specified string.\nResult type: String\nExample:\n RETURN rtrim(\" basketball player \");\nleft()\nleft() returns a substring consisting of several characters from the leading of a string.\nSyntax: left(,)\nstring: A specified string.\ncount: The number of characters from the leading of the string. If the string is shorter than count, the system returns the string itself.\nResult type: String\nExample:\n RETURN left(\"basketball_player\",6);\nright()\nright() returns a substring consisting of several characters from the trailing of a string.\nSyntax: right(,)\nstring: A specified string.\ncount: The number of characters from the trailing of the string. If the string is shorter than count, the system returns the string itself.\nResult type: String\nExample:\n RETURN right(\"basketball_player\",6);\nlpad()\nlpad() pads a specified string from the left-side to the specified length and returns the result string.\nSyntax: lpad(,,)\nstring: A specified string.\ncount: The length of the string after it has been left-padded. If the length is less than that of string, only the length of string characters from front to back will be returned.\nletters: A string to be padding from the leading.\nResult type: String\nExample:\n RETURN lpad(\"abcd\",10,\"b\");\n RETURN lpad(\"abcd\",3,\"b\");\nrpad()\nrpad() pads a specified string from the right-side to the specified length and returns the result string.\nSyntax: rpad(,,)\nstring: A specified string.\ncount: The length of the string after it has been right-padded. If the length is less than that of string, only the length of string characters from front to back will be returned.\nletters: A string to be padding from the trailing.\nResult type: String\nExample:\n RETURN rpad(\"abcd\",10,\"b\");\n RETURN rpad(\"abcd\",3,\"b\");\nsubstr() and substring()\nsubstr() and substring() return a substring extracting count characters starting from the specified position pos of a specified string.\nSyntax: substr(,,), substring(,,)\nstring: A specified string.\npos: The position of starting extract (character index). Data type is int.\ncount: The number of characters extracted from the start position onwards.\nResult type: String\nExplanations for the return of substr() and substring()\nIf pos is 0, it extracts from the specified string leading (including the first character).\nIf pos is greater than the maximum string index, an empty string is returned.\nIf pos is a negative number, BAD_DATA is returned.\nIf count is omitted, the function returns the substring starting at the position given by pos and extending to the end of the string.\nIf count is 0, an empty string is returned.\nUsing NULL as any of the argument of substr() will cause an issue.\nExample:\n RETURN substr(\"abcdefg\",2,4);\n RETURN substr(\"abcdefg\",0,4);\n RETURN substr(\"abcdefg\",2);\nreverse()\nreverse() returns a string in reverse order.\nSyntax: reverse()\nstring: A specified string.\nResult type: String\nExample:\n RETURN reverse(\"abcdefg\");\nreplace()\nreplace() replaces string a in a specified string with string b.\nSyntax: replace(,,)\nstring: A specified string.\nsubstr_a: String a.\nstring_b: String b.\nResult type: String\nExample:\n RETURN replace(\"abcdefg\",\"cd\",\"AAAAA\");\nsplit()\nsplit() splits a specified string at string b and returns a list of strings.\nSyntax: split(,)\nstring: A specified string.\nsubstr: String b.\nResult type: List\nExample:\n RETURN split(\"basketballplayer\",\"a\");\nconcat()\nconcat() returns strings concatenated by all strings.\nSyntax: concat(,,...)\nThe function requires at least two or more strings. If there is only one string, the string itself is returned.\nIf any one of the strings is NULL, NULL is returned.\nResult type: String\nExample:\n//This example concatenates 1, 2, and 3.\n RETURN concat(\"1\",\"2\",\"3\") AS r;\n//In this example, one of the string is NULL.\n RETURN concat(\"1\",\"2\",NULL) AS r;\n GO FROM \"player100\" over follow \\\n YIELD concat(src(edge), properties($^).age, properties($$).name, properties(edge).degree) AS A;\nconcat_ws()\nconcat_ws() returns strings concatenated by all strings that are delimited with a separator.\nSyntax: concat_ws(,,,... )\nThe function requires at least two or more strings.\nIf the separator is NULL, the concat_ws() function returns NULL.\nIf the separator is not NULL and there is only one string, the string itself is returned.\nIf there is a NULL in the strings, NULL is ignored during the concatenation.\nExample:\n//This example concatenates a, b, and c with the separator +.\n RETURN concat_ws(\"+\",\"a\",\"b\",\"c\") AS r;\n//In this example, the separator is NULL.\nneubla> RETURN concat_ws(NULL,\"a\",\"b\",\"c\") AS r;\n//In this example, the separator is + and there is a NULL in the strings.\n RETURN concat_ws(\"+\",\"a\",NULL,\"b\",\"c\") AS r;\n//In this example, the separator is + and there is only one string.\n RETURN concat_ws(\"+\",\"a\") AS r;\n GO FROM \"player100\" over follow \\\n YIELD concat_ws(\" \",src(edge), properties($^).age, properties($$).name, properties(edge).degree) AS A;\nextract()\nextract() uses regular expression matching to retrieve a single substring or all substrings from a string.\nSyntax: extract(,\"\")\nstring: A specified string\nregular_expression: A regular expression\nResult type: List\nExample:\n MATCH (a:player)-[b:serve]-(c:team{name: \"Lakers\"}) \\\n WHERE a.player.age > 45 \\\n RETURN extract(a.player.name, \"\\\\w+\") AS result;\n MATCH (a:player)-[b:serve]-(c:team{name: \"Lakers\"}) \\\n WHERE a.player.age > 45 \\\n RETURN extract(a.player.name, \"hello\") AS result;\njson_extract()\njson_extract() converts the specified JSON string to a map.\nSyntax: extract()\nstring:A specified string, must be JSON string.\nResult type: Map\nExample:\n YIELD json_extract('{\"a\": 1, \"b\": {}, \"c\": {\"d\": true}}') AS result;","url":"6.functions-and-expressions/2.string/","type":"doc"},{"title":"Built-in date and time functions","content":"NebulaGraph supports the following built-in date and time functions:\nFunction\nDescription\nint now()\nReturns the current timestamp of the system.\ntimestamp timestamp()\nReturns the current timestamp of the system.\ndate date()\nReturns the current UTC date based on the current system.\ntime time()\nReturns the current UTC time based on the current system.\ndatetime datetime()\nReturns the current UTC date and time based on the current system.\nmap duration()\nReturns the period of time. It can be used to calculate the specified time.\nFor more information, see Date and time types.\nExamples\n RETURN now(), timestamp(), date(), time(), datetime();","url":"6.functions-and-expressions/3.date-and-time/","type":"doc"},{"title":"Schema-related functions","content":"This topic describes the schema-related functions supported by NebulaGraph. There are two types of schema-related functions, one for native nGQL statements and the other for openCypher-compatible statements.\nFor nGQL statements\nThe following functions are available in YIELD and WHERE clauses of nGQL statements.\nid(vertex)\nid(vertex) returns the ID of a vertex.\nSyntax: id(vertex)\nResult type: Same as the vertex ID.\nExample:\n LOOKUP ON player WHERE player.age > 45 YIELD id(vertex);\nproperties(vertex)\nproperties(vertex) returns the properties of a vertex.\nSyntax: properties(vertex)\nResult type: Map\nExample:\n LOOKUP ON player WHERE player.age > 45 \\\n YIELD properties(vertex);\nYou can also use the property reference symbols ($^ and $$) instead of the vertex field in the properties() function to get all properties of a vertex.\n$^ represents the data of the starting vertex at the beginning of exploration. For example, in GO FROM \"player100\" OVER follow reversely YIELD properties($^), $^ refers to the vertex player100.\n$$ represents the data of the end vertex at the end of exploration.\nproperties($^) and properties($$) are generally used in GO statements. For more information, see Property reference.\nproperties(edge)\nproperties(edge) returns the properties of an edge.\nSyntax: properties(edge)\nResult type: Map\nExample:\n GO FROM \"player100\" OVER follow \\\n YIELD properties(edge);\ntype(edge)\ntype(edge) returns the edge type of an edge.\nSyntax: type(edge)\nResult type: String\nExample:\n GO FROM \"player100\" OVER follow \\\n YIELD src(edge), dst(edge), type(edge), rank(edge);\nsrc(edge)\nsrc(edge) returns the source vertex ID of an edge.\nSyntax: src(edge)\nResult type: Same as the vertex ID.\nExample:\n GO FROM \"player100\" OVER follow \\\n YIELD src(edge), dst(edge);\ndst(edge)\ndst(edge) returns the destination vertex ID of an edge.\nSyntax: dst(edge)\nResult type: Same as the vertex ID.\nExample:\n GO FROM \"player100\" OVER follow \\\n YIELD src(edge), dst(edge);\nrank(edge)\nrank(edge) returns the rank value of an edge.\nSyntax: rank(edge)\nResult type: Int\nExample:\n GO FROM \"player100\" OVER follow \\\n YIELD src(edge), dst(edge), rank(edge);\nvertex\nvertex returns the information of vertices, including VIDs, tags, properties, and values. You need to use AS to set the alias.\nSyntax: vertex\nExample:\n LOOKUP ON player WHERE player.age > 45 YIELD vertex AS v;\nedge\nedge returns the information of edges, including edge types, source vertices, destination vertices, ranks, properties, and values. You need to use AS to set the alias.\nSyntax: edge\nExample:\n GO FROM \"player100\" OVER follow YIELD edge AS e;\nvertices\nvertices returns the information of vertices in a subgraph. For more information, see GET SUBGRAPH.\nedges\nedges returns the information of edges in a subgraph. For more information, see GET SUBGRAPH.\npath\npath returns the information of a path. For more information, see FIND PATH.\nFor statements compatible with openCypher\nThe following functions are available in RETURN and WHERE clauses of openCypher-compatible statements.\nid()\nid() returns the ID of a vertex.\nSyntax: id()\nResult type: Same as the vertex ID.\nExample:\n MATCH (v:player) RETURN id(v); \n...\ntags() and labels()\ntags() and labels() return the Tag of a vertex.\nSyntax: tags(), labels()\nResult type: List\nExample:\n MATCH (v) WHERE id(v) == \"player100\" \\\n RETURN tags(v);\nproperties()\nproperties() returns the properties of a vertex or an edge.\nSyntax: properties()\nResult type: Map\nExample:\n MATCH (v:player)-[e:follow]-() RETURN properties(v),properties(e);\n...\ntype()\ntype() returns the edge type of an edge.\nSyntax: type()\nResult type: String\nExample:\n MATCH (v:player{name:\"Tim Duncan\"})-[e]->() \\\n RETURN type(e);\ntypeid()\ntypeid() returns the internal ID value of the Edge type of the edge, which can be used to determine the direction by positive or negative.\nSyntax: typeid()\nResult type: Int\nExample:\n MATCH (v:player)-[e:follow]-(v2) RETURN e,typeid(e), \\\n CASE WHEN typeid(e) > 0 \\\n THEN \"Forward\" ELSE \"Reverse\" END AS direction \\\n LIMIT 5;\nsrc()\nsrc() returns the source vertex ID of an edge.\nSyntax: src()\nResult type: Same as the vertex ID.\nExample:\n MATCH ()-[e]->(v:player{name:\"Tim Duncan\"}) \\\n RETURN src(e);\n...\ndst()\ndst() returns the destination vertex ID of an edge.\nSyntax: dst()\nResult type: Same as the vertex ID.\nExample:\n MATCH (v:player{name:\"Tim Duncan\"})-[e]->() \\\n RETURN dst(e);\nstartNode()\nstartNode() visits a path and returns its information of source vertex ID, including VIDs, tags, properties, and values.\nSyntax: startNode()\nExample:\n MATCH p = (a :player {name : \"Tim Duncan\"})-[r:serve]-(t) \\\n RETURN startNode(p);\nendNode()\nendNode() visits a path and returns its information of destination vertex ID, including VIDs, tags, properties, and values.\nSyntax: endNode()\nExample:\n MATCH p = (a :player {name : \"Tim Duncan\"})-[r:serve]-(t) \\\n RETURN endNode(p);\nrank()\nrank() returns the rank value of an edge.\nSyntax: rank()\nResult type: Int\nExample:\n MATCH (v:player{name:\"Tim Duncan\"})-[e]->() \\\n RETURN rank(e);","url":"6.functions-and-expressions/4.schema/","type":"doc"},{"title":"List functions","content":"This topic describes the list functions supported by NebulaGraph. Some of the functions have different syntax in native nGQL statements and openCypher-compatible statements.\nPrecautions\nLike SQL, the position index in nGQL starts from 1, while in the C language it starts from 0.\nGeneral\nrange()\nrange() returns the list containing all the fixed-length steps in [start,end].\nSyntax: range(start, end [, step])\nstep: Optional parameters. step is 1 by default.\nResult type: List\nExample:\n RETURN range(1,9,2);\nreverse()\nreverse() returns the list reversing the order of all elements in the original list.\nSyntax: reverse()\nResult type: List\nExample:\n WITH [NULL, 4923, 'abc', 521, 487] AS ids \\\n RETURN reverse(ids);\ntail()\ntail() returns all the elements of the original list, excluding the first one.\nSyntax: tail()\nResult type: List\nExample:\n WITH [NULL, 4923, 'abc', 521, 487] AS ids \\\n RETURN tail(ids);\nhead()\nhead() returns the first element of a list.\nSyntax: head()\nResult type: Same as the element in the original list.\nExample:\n WITH [NULL, 4923, 'abc', 521, 487] AS ids \\\n RETURN head(ids);\nlast()\nlast() returns the last element of a list.\nSyntax: last()\nResult type: Same as the element in the original list.\nExample:\n WITH [NULL, 4923, 'abc', 521, 487] AS ids \\\n RETURN last(ids);\nreduce()\nreduce() applies an expression to each element in a list one by one, chains the result to the next iteration by taking it as the initial value, and returns the final result. This function iterates each element e in the given list, runs the expression on e, accumulates the result with the initial value, and store the new result in the accumulator as the initial value of the next iteration. It works like the fold or reduce method in functional languages such as Lisp and Scala.\nSyntax: reduce( = , IN | )\naccumulator: A variable that will hold the accumulated results as the list is iterated.\ninitial: An expression that runs once to give an initial value to the accumulator.\nvariable: A variable in the list that will be applied to the expression successively.\nlist: A list or a list of expressions.\nexpression: This expression will be run on each element in the list once and store the result value in the accumulator.\nResult type: Depends on the parameters provided, along with the semantics of the expression.\nExample:\n RETURN reduce(totalNum = -4 * 5, n IN [1, 2] | totalNum + n * 2) AS r;\n MATCH p = (n:player{name:\"LeBron James\"})<-[:follow]-(m) \\\n RETURN nodes(p)[0].player.age AS src1, nodes(p)[1].player.age AS dst2, \\\n reduce(totalAge = 100, n IN nodes(p) | totalAge + n.player.age) AS sum;\n LOOKUP ON player WHERE player.name == \"Tony Parker\" YIELD id(vertex) AS VertexID \\\n | GO FROM $-.VertexID over follow \\\n WHERE properties(edge).degree != reduce(totalNum = 5, n IN range(1, 3) | properties($$).age + totalNum + n) \\\n YIELD properties($$).name AS id, properties($$).age AS age, properties(edge).degree AS degree;\nFor nGQL statements\nkeys()\nkeys() returns a list containing the string representations for all the property names of vertices or edges.\nSyntax: keys({vertex | edge})\nResult type: List\nExample:\n LOOKUP ON player \\\n WHERE player.age > 45 \\\n YIELD keys(vertex);\nlabels()\nlabels() returns the list containing all the tags of a vertex.\nSyntax: labels(verte)\nResult type: List\nExample:\n FETCH PROP ON * \"player101\", \"player102\", \"team204\" \\\n YIELD labels(vertex);\nFor statements compatible with openCypher\nkeys()\nkeys() returns a list containing the string representations for all the property names of vertices, edges, or maps.\nSyntax: keys()\nResult type: List\nExample:\n MATCH (v:player{name:\"Tim Duncan\"})-[e]->() \\\n RETURN keys(e);\nlabels()\nlabels() returns the list containing all the tags of a vertex.\nSyntax: labels()\nResult type: List\nExample:\n MATCH (v)-[e:serve]->() \\\n WHERE id(v)==\"player100\" \\\n RETURN labels(v);\nnodes()\nnodes() returns the list containing all the vertices in a path.\nSyntax: nodes()\nResult type: List\nExample:\n MATCH p=(v:player{name:\"Tim Duncan\"})-->(v2) \\\n RETURN nodes(p);\nrelationships()\nrelationships() returns the list containing all the relationships in a path.\nSyntax: relationships()\nResult type: List\nExample:\n MATCH p=(v:player{name:\"Tim Duncan\"})-->(v2) \\\n RETURN relationships(p);","url":"6.functions-and-expressions/6.list/","type":"doc"},{"title":"Type conversion functions","content":"This topic describes the type conversion functions supported by NebulaGraph.\ntoBoolean()\ntoBoolean() converts a string value to a boolean value.\nSyntax: toBoolean()\nResult type: Bool\nExample:\n UNWIND [true, false, 'true', 'false', NULL] AS b \\\n RETURN toBoolean(b) AS b;\ntoFloat()\ntoFloat() converts an integer or string value to a floating point number.\nSyntax: toFloat()\nResult type: Float\nExample:\n RETURN toFloat(1), toFloat('1.3'), toFloat('1e3'), toFloat('not a number');\ntoString()\ntoString() converts non-compound types of data, such as numbers, booleans, and so on, to strings.\nSyntax: toString()\nResult type: String\nExample:\n RETURN toString(9669) AS int2str, toString(null) AS null2str;\ntoInteger()\ntoInteger() converts a floating point or string value to an integer value.\nSyntax: toInteger()\nResult type: Int\nExample:\n RETURN toInteger(1), toInteger('1'), toInteger('1e3'), toInteger('not a number');\ntoSet()\ntoSet() converts a list or set value to a set value.\nSyntax: toSet()\nResult type: Set\nExample:\n RETURN toSet(list[1,2,3,1,2]) AS list2set;\nhash()\nhash() returns the hash value of the argument. The argument can be a number, a string, a list, a boolean, null, or an expression that evaluates to a value of the preceding data types.\nThe source code of the hash() function (MurmurHash2), seed (0xc70f6907UL), and other parameters can be found in MurmurHash2.h.\nFor Java, the hash function operates as follows.\nMurmurHash2.hash64(\"to_be_hashed\".getBytes(),\"to_be_hashed\".getBytes().length, 0xc70f6907)\nSyntax: hash()\nResult type: Int\nExample:\n RETURN hash(\"abcde\");\n YIELD hash([1,2,3]);\n YIELD hash(NULL);\n YIELD hash(toLower(\"HELLO NEBULA\"));","url":"6.functions-and-expressions/16.type-conversion/","type":"doc"},{"title":"Conditional expressions","content":"This topic describes the conditional functions supported by NebulaGraph.\nCASE\nThe CASE expression uses conditions to filter the parameters. nGQL provides two forms of CASE expressions just like openCypher: the simple form and the generic form.\nThe CASE expression will traverse all the conditions. When the first condition is met, the CASE expression stops reading the conditions and returns the result. If no conditions are met, it returns the result in the ELSE clause. If there is no ELSE clause and no conditions are met, it returns NULL.\nThe simple form of CASE expressions\nSyntax\nCASE \nWHEN THEN \n[WHEN ...]\n[ELSE ]\nEND\nParameter\nDescription\ncomparer\nA value or a valid expression that outputs a value. This value is used to compare with the value.\nvalue\nIt will be compared with the comparer. If the value matches the comparer, then this condition is met.\nresult\nThe result is returned by the CASE expression if the value matches the comparer.\ndefault\nThe default is returned by the CASE expression if no conditions are met.\nExamples\n RETURN \\\n CASE 2+3 \\\n WHEN 4 THEN 0 \\\n WHEN 5 THEN 1 \\\n ELSE -1 \\\n END \\\n AS result;\n GO FROM \"player100\" OVER follow \\\n YIELD properties($$).name AS Name, \\\n CASE properties($$).age > 35 \\\n WHEN true THEN \"Yes\" \\\n WHEN false THEN \"No\" \\\n ELSE \"Nah\" \\\n END \\\n AS Age_above_35;\nThe generic form of CASE expressions\nSyntax\nCASE\nWHEN THEN \n[WHEN ...]\n[ELSE ]\nEND\nParameter\nDescription\ncondition\nIf the condition is evaluated as true, the result is returned by the CASE expression.\nresult\nThe result is returned by the CASE expression if the condition is evaluated as true.\ndefault\nThe default is returned by the CASE expression if no conditions are met.\nExamples\n YIELD \\\n CASE WHEN 4 > 5 THEN 0 \\\n WHEN 3+4==7 THEN 1 \\\n ELSE 2 \\\n END \\\n AS result;\n MATCH (v:player) WHERE v.player.age > 30 \\\n RETURN v.player.name AS Name, \\\n CASE \\\n WHEN v.player.name STARTS WITH \"T\" THEN \"Yes\" \\\n ELSE \"No\" \\\n END \\\n AS Starts_with_T;\nDifferences between the simple form and the generic form\nTo avoid the misuse of the simple form and the generic form, it is important to understand their differences. The following example can help explain them.\n GO FROM \"player100\" OVER follow \\\n YIELD properties($$).name AS Name, properties($$).age AS Age, \\\n CASE properties($$).age \\\n WHEN properties($$).age > 35 THEN \"Yes\" \\\n ELSE \"No\" \\\n END \\\n AS Age_above_35;\nThe preceding GO query is intended to output Yes when the player's age is above 35. However, in this example, when the player's age is 36, the actual output is not as expected: It is No instead of Yes.\nThis is because the query uses the CASE expression in the simple form, and a comparison between the values of $$.player.age and $$.player.age > 35 is made. When the player age is 36:\nThe value of $$.player.age is 36. It is an integer.\n$$.player.age > 35 is evaluated to be true. It is a boolean.\nThe values of $$.player.age and $$.player.age > 35 do not match. Therefore, the condition is not met and No is returned.\ncoalesce()\ncoalesce() returns the first not null value in all expressions.\nSyntax: coalesce([,...])\nResult type: Same as the original element.\nExample:\n RETURN coalesce(null,[1,2,3]) as result;\n RETURN coalesce(null) as result;","url":"6.functions-and-expressions/5.conditional-expressions/","type":"doc"},{"title":"Predicate functions","content":"Predicate functions return true or false. They are most commonly used in WHERE clauses.\nNebulaGraph supports the following predicate functions:\nFunctions\nDescription\nexists()\nReturns true if the specified property exists in the vertex, edge or map. Otherwise, returns false.\nany()\nReturns true if the specified predicate holds for at least one element in the given list. Otherwise, returns false.\nall()\nReturns true if the specified predicate holds for all elements in the given list. Otherwise, returns false.\nnone()\nReturns true if the specified predicate holds for no element in the given list. Otherwise, returns false.\nsingle()\nReturns true if the specified predicate holds for exactly one of the elements in the given list. Otherwise, returns false.\nSyntax\n( IN WHERE )\nExamples\n RETURN any(n IN [1, 2, 3, 4, 5, NULL] \\\n WHERE n > 2) AS r;\n RETURN single(n IN range(1, 5) \\\n WHERE n == 3) AS r;\n RETURN none(n IN range(1, 3) \\\n WHERE n == 0) AS r;\n WITH [1, 2, 3, 4, 5, NULL] AS a \\\n RETURN any(n IN a WHERE n > 2);\n MATCH p = (n:player{name:\"LeBron James\"})<-[:follow]-(m) \\\n RETURN nodes(p)[0].player.name AS n1, nodes(p)[1].player.name AS n2, \\\n all(n IN nodes(p) WHERE n.player.name NOT STARTS WITH \"D\") AS b;\n MATCH p = (n:player{name:\"LeBron James\"})-[:follow]->(m) \\\n RETURN single(n IN nodes(p) WHERE n.player.age > 40) AS b;\n MATCH (n:player) \\\n RETURN exists(n.player.id), n IS NOT NULL;\n...\n MATCH (n:player) \\\n WHERE exists(n['name']) RETURN n;\n...","url":"6.functions-and-expressions/8.predicate/","type":"doc"},{"title":"Geography functions","content":"Geography functions are used to generate or perform operations on the value of the geography data type.\nFor descriptions of the geography data types, see Geography.\nDescriptions\nFunction\nReturn Type\nDescription\nST_Point(longitude, latitude)\nGEOGRAPHY\nCreates the geography that contains a point.\nST_GeogFromText(wkt_string)\nGEOGRAPHY\nReturns the geography corresponding to the input WKT string.\nST_ASText(geography)\nSTRING\nReturns the WKT string of the input geography.\nST_Centroid(geography)\nGEOGRAPHY\nReturns the centroid of the input geography in the form of the single point geography.\nST_ISValid(geography)\nBOOL\nReturns whether the input geography is valid.\nST_Intersects(geography_1, geography_2)\nBOOL\nReturns whether geography_1 and geography_2 have intersections.\nST_Covers(geography_1, geography_2)\nBOOL\nReturns whether geography_1 completely contains geography_2. If there is no point outside geography_1 in geography_2, return True.\nST_CoveredBy(geography_1, geography_2)\nBOOL\nReturns whether geography_2 completely contains geography_1.If there is no point outside geography_2 in geography_1, return True.\nST_DWithin(geography_1, geography_2, distance)\nBOOL\nIf the distance between one point (at least) in geography_1 and one point in geography_2 is less than or equal to the distance specified by the distance parameter (measured by meters), return True.\nST_Distance(geography_1, geography_2)\nFLOAT\nReturns the smallest possible distance (measured by meters) between two non-empty geographies.\nS2_CellIdFromPoint(point_geography)\nINT\nReturns the S2 Cell ID that covers the point geography.\nS2_CoveringCellIds(geography)\nARRAY\nReturns an array of S2 Cell IDs that cover the input geography.\nExamples\n RETURN ST_ASText(ST_Point(1,1));\n RETURN ST_ASText(ST_GeogFromText(\"POINT(3 8)\"));\n RETURN ST_ASTEXT(ST_Centroid(ST_GeogFromText(\"LineString(0 1,1 0)\")));\n RETURN ST_ISValid(ST_GeogFromText(\"POINT(3 8)\"));\n RETURN ST_Intersects(ST_GeogFromText(\"LineString(0 1,1 0)\"),ST_GeogFromText(\"LineString(0 0,1 1)\"));\n RETURN ST_Covers(ST_GeogFromText(\"POLYGON((0 0,10 0,10 10,0 10,0 0))\"),ST_Point(1,2));\n RETURN ST_CoveredBy(ST_Point(1,2),ST_GeogFromText(\"POLYGON((0 0,10 0,10 10,0 10,0 0))\"));\n RETURN ST_dwithin(ST_GeogFromText(\"Point(0 0)\"),ST_GeogFromText(\"Point(10 10)\"),20000000000.0);\n RETURN ST_Distance(ST_GeogFromText(\"Point(0 0)\"),ST_GeogFromText(\"Point(10 10)\"));\n RETURN S2_CellIdFromPoint(ST_GeogFromText(\"Point(1 1)\"));\n RETURN S2_CoveringCellIds(ST_GeogFromText(\"POLYGON((0 1, 1 2, 2 3, 0 1))\"));","url":"6.functions-and-expressions/14.geo/","type":"doc"},{"title":"Overview of NebulaGraph general query statements","content":"This topic provides an overview of the general categories of query statements in NebulaGraph and outlines their use cases.\nBackground\nNebulaGraph stores data in the form of vertices and edges. Each vertex can have zero or more tags and each edge has exactly one edge type. Tags define the type of a vertex and describe its properties, while edge types define the type of an edge and describe its properties. When querying, you can limit the scope of the query by specifying the tag of a vertex or the type of an edge. For more information, see Patterns.\nCategories\nThe primary query statements in NebulaGraph fall into the following categories:\nFETCH PROP ON\nLOOKUP ON\nGO\nMATCH\nFIND PATH\nGET SUBGRAPH\nSHOW\nFETCH PROP ON and LOOKUP ON statements are primarily for basic data queries, GO and MATCH for more intricate queries and graph traversals, FIND PATH and GET SUBGRAPH for path and subgraph queries, and SHOW for retrieving database metadata.\nUsage and use cases\nFETCH PROP ON\nUsage: Retrieve properties of a specified vertex or edge.\nUse case: Knowing the specific vertex or edge ID and wanting to retrieve its properties.\nNote:\nMust specify the ID of the vertex or edge.\nMust specify the tag of the vertex or the edge type of the edge.\nMust use the YIELD clause to specify the returned properties.\nExample:\nFETCH PROP ON player \"player100\" YIELD properties(vertex);\n --+--- ----+----- -------+----------\n | | |\n | | |\n | | +--------- Returns all properties under the player tag of the vertex.\n | |\n | +----------------- Retrieves from the vertex \"player100\".\n +--------------------------- Retrieves properties under the player tag.\nFor more information, see FETCH PROP ON.\nLOOKUP ON\nUsage: Index-based querying of vertex or edge IDs.\nUse case: Finding vertex or edge IDs based on property values.\nNote:\n- Must pre-define indexes for the tag, edge type, or property.\n- Must specify the tag of the vertex or the edge type of the edge.\n- Must use the YIELD clause to specify the returned IDs.\nExample:\nLOOKUP ON player WHERE player.name == \"Tony Parker\" YIELD id(vertex);\n --+--- ------------------+--------------- ---+------\n | | |\n | | |\n | | +---- Returns the VID of the retrieved vertex.\n | |\n | +------------ Filtering is based on the value of the property name.\n +----------------------------------- Queries based on the player tag.\nFor more information, see LOOKUP ON.\nGO\nUsage: Traverse the graph based on a given vertex and return information about the starting vertex, edges, or target vertices as needed.\nUse case: Complex graph traversals, such as finding friends of a vertex, friends' friends, etc.\nNote:\n- Use property reference symbols ($^ and $$) to return properties of the starting or target vertices, e.g., YIELD $^.player.name.\n- Use the functions properties($^) and properties($$) to return all properties of the starting or target vertices. Specify property names in the function to return specific properties, e.g., YIELD properties($^).name.\n- Use the functions src(edge) and dst(edge) to return the starting or destination vertex ID of an edge, e.g., YIELD src(edge).\nExample:\nGO 3 STEPS FROM \"player102\" OVER follow YIELD dst(edge);\n-----+--- --+------- -+---- ---+-----\n | | | |\n | | | |\n | | | +--------- Returns the destination vertex of the last hop.\n | | |\n | | +------ Traverses out via the edge follow.\n | |\n | +--------------------- Starts from \"player102\".\n +---------------------------------- Traverses 3 steps.\nFor more information, see GO.\nMATCH\nUsage: Execute complex graph pattern matching queries.\nUse case: Complex graph pattern matching, such as finding combinations of vertices and edges that satisfy a specific pattern.\nNote:\nMATCH statements are compatible with the OpenCypher syntax but with some differences:\nUse == for equality instead of =, e.g., WHERE player.name == \"Tony Parker\".\nWhen referencing properties of vertices, you need to specify the vertex's tag, e.g., YIELD player.name.\nIntroduces the WHERE id(v) == \"player100\" syntax.\nMust use the RETURN clause to specify what information to return.\nExample:\nMATCH (v:player{name:\"Tim Duncan\"})-->(v2:player) \\\n RETURN v2.player.name AS Name;\nFor more information, see MATCH.\nFIND PATH\nUsage: Query paths between given starting and target vertices or query properties of vertices and edges along paths.\nUse case: Querying paths between two vertices.\nNote: Must use the YIELD clause to specify returned information.\nExample:\nFIND SHORTEST PATH FROM \"player102\" TO \"team204\" OVER * YIELD path AS p;\n-------+----- -------+---------------- ---+-- ----+----\n | | | |\n | | | |\n | | | +---------- Returns the path as 'p'.\n | | |\n | | +----------- Travels outwards via all types of edges.\n | | \n | |\n | +------------------ From the given starting and target VIDs. \n +--------------------------- Retrieves the shortest path.\nFor more information, see FIND PATH.\nGET SUBGRAPH\nUsage: Extract a portion of the graph that satisfies specific conditions or query properties of vertices and edges in the subgraph.\nUse case: Analyzing structures of the graph or specific regions, such as extracting the social network subgraph of a person or the transportation network subgraph of an area.\nNote: Must use the YIELD clause to specify returned information.\nExample:\nGET SUBGRAPH 5 STEPS FROM \"player101\" YIELD VERTICES AS nodes, EDGES AS relationships;\n -----+- -----+-------- ------------------------+----------------\n | | |\n | | |\n | +------- Starts from \"player101\". +------------ Returns all vertices and edges.\n +----------------- Gets exploration of 5 steps \nFor more information, see GET SUBGRAPH.\nSHOW\nSHOW statements are mainly used to obtain metadata information from the database, not for retrieving the actual data stored in the database. These statements are typically used to query the structure and configuration of the database.\nStatement\nSyntax\nExample\nDescription\nSHOW CHARSET\nSHOW CHARSET\nSHOW CHARSET\nShows the available character sets.\nSHOW COLLATION\nSHOW COLLATION\nSHOW COLLATION\nShows the collations supported by NebulaGraph.\nSHOW CREATE SPACE\nSHOW CREATE SPACE \nSHOW CREATE SPACE basketballplayer\nShows the creating statement of the specified graph space.\nSHOW CREATE TAG/EDGE\nSHOW CREATE {TAG | EDGE }\nSHOW CREATE TAG player\nShows the basic information of the specified tag.\nSHOW HOSTS\nSHOW HOSTS [GRAPH | STORAGE | META]\nSHOW HOSTSSHOW HOSTS GRAPH\nShows the host and version information of Graph Service, Storage Service, and Meta Service.\nSHOW INDEX STATUS\nSHOW {TAG | EDGE} INDEX STATUS\nSHOW TAG INDEX STATUS\nShows the status of jobs that rebuild native indexes, which helps check whether a native index is successfully rebuilt or not.\nSHOW INDEXES\nSHOW {TAG | EDGE} INDEXES\nSHOW TAG INDEXES\nShows the names of existing native indexes.\nSHOW PARTS\nSHOW PARTS []\nSHOW PARTS\nShows the information of a specified partition or all partitions in a graph space.\nSHOW ROLES\nSHOW ROLES IN \nSHOW ROLES in basketballplayer\nShows the roles that are assigned to a user account.\nSHOW SNAPSHOTS\nSHOW SNAPSHOTS\nSHOW SNAPSHOTS\nShows the information of all the snapshots.\nSHOW SPACES\nSHOW SPACES\nSHOW SPACES\nShows existing graph spaces in NebulaGraph.\nSHOW STATS\nSHOW STATS\nSHOW STATS\nShows the statistics of the graph space collected by the latest STATS job.\nSHOW TAGS/EDGES\nSHOW TAGS | EDGES\nSHOW TAGS,SHOW EDGES\nShows all the tags in the current graph space.\nSHOW USERS\nSHOW USERS\nSHOW USERS\nShows the user information.\nSHOW SESSIONS\nSHOW SESSIONS\nSHOW SESSIONS\nShows the information of all the sessions.\nSHOW SESSIONS\nSHOW SESSION \nSHOW SESSION 1623304491050858\nShows a specified session with its ID.\nSHOW QUERIES\nSHOW [ALL] QUERIES\nSHOW QUERIES\nShows the information of working queries in the current session.\nSHOW META LEADER\nSHOW META LEADER\nSHOW META LEADER\nShows the information of the leader in the current Meta cluster.\nCompound queries\nQuery statements in NebulaGraph can be combined to achieve more complex queries.\nWhen referencing the results of a subquery in a compound statement, you need to create an alias for the result and use the pipe symbol(|) to pass it to the next subquery. Use $- in the next subquery to reference the alias of that result. See Pipe Symbol for details.\nExample:\n GO FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS dstid, properties($$).name AS Name | \\\n GO FROM $-.dstid OVER follow YIELD dst(edge);\nThe pipe symbol | is applicable only in nGQL and cannot be used in OpenCypher statements. If you need to perform compound queries using MATCH statements, you can use the WITH clause.\nExample:\n MATCH (v:player)-->(v2:player) \\\n WITH DISTINCT v2 AS v2, v2.player.age AS Age \\\n ORDER BY Age \\\n WHERE Age<25 \\\n RETURN v2.player.name AS Name, Age;\nMore information\nnGQL command cheatsheet","url":"7.general-query-statements/1.general-query-statements-overview/","type":"doc"},{"title":"MATCH","content":"The MATCH statement provides pattern-based search functionality, allowing you to retrieve data that matches one or more patterns in NebulaGraph. By defining one or more patterns, you can search for data that matches the patterns in NebulaGraph. Once the matching data is retrieved, you can use the RETURN clause to return it as a result.\nThe examples in this topic use the basketballplayer dataset as the sample dataset.\nSyntax\nThe syntax of MATCH is relatively more flexible compared with that of other query statements such as GO or LOOKUP. The path type of the MATCH statement is trail. That is, only vertices can be repeatedly visited in the graph traversal. Edges cannot be repeatedly visited. For details, see path. But generally, it can be summarized as follows.\nMATCH [] RETURN [];\npattern: The MATCH statement supports matching one or multiple patterns. Multiple patterns are separated by commas (,). For example: (a)-[]->(b),(c)-[]->(d). For the detailed description of patterns, see Patterns. \nclause_1: The WHERE, WITH, UNWIND, and OPTIONAL MATCH clauses are supported, and the MATCH clause can also be used.\noutput: Define the list name for the output results to be returned. You can use AS to set an alias for the list.\nclause_2: The ORDER BY and LIMIT clauses are supported.\nNotes\nAvoid full table scans, as they may result in decreased query performance, and if there is insufficient memory during a full table scan, the query may fail, and the system will report an error. It is recommended to use queries with filter conditions or specifying tags and edge types, such as v:player and v.player.name in the statement MATCH (v:player) RETURN v.player.name AS Name.\nYou can create an index for a tag, edge type, or a specific property of a tag or edge type to improve query performance. For example, you can create an index for the player tag or the name property of the player tag. For more information about the usage and considerations for indexes, see Must-read for using indexes.\nThe MATCH statement cannot query dangling edges.\nUsing patterns in MATCH statements\nMatch vertices\nYou can use a user-defined variable in a pair of parentheses to represent a vertex in a pattern. For example: (v).\n MATCH (v) \\\n RETURN v \\\n LIMIT 3;\nMatch tags\nYou can specify a tag with : after the vertex in a pattern.\n MATCH (v:player) \\\n RETURN v;\n...\nTo match vertices with multiple tags, use colons (:).\n CREATE TAG actor (name string, age int);\n INSERT VERTEX actor(name, age) VALUES \"player100\":(\"Tim Duncan\", 42);\n MATCH (v:player:actor) \\\n RETURN v \\\nMatch vertex properties\nYou can specify a vertex property with {: } after the tag in a pattern.\n# The following example uses the name property to match a vertex.\n MATCH (v:player{name:\"Tim Duncan\"}) \\\n RETURN v;\nThe WHERE clause can do the same thing:\n MATCH (v:player) \\\n WHERE v.player.name == \"Tim Duncan\" \\\n RETURN v;\nUse the WHERE clause to directly get all the vertices with the vertex property value Tim Duncan.\n MATCH (v) \\\n WITH v, properties(v) as props, keys(properties(v)) as kk \\\n WHERE [i in kk where props[i] == \"Tim Duncan\"] \\\n RETURN v;\n WITH ['Tim Duncan', 'Yao Ming'] AS names \\\n MATCH (v1:player)-->(v2:player) \\\n WHERE v1.player.name in names \\\n return v1, v2;\nMatch VIDs\nYou can use the VID to match a vertex. The id() function can retrieve the VID of a vertex.\n MATCH (v) \\\n WHERE id(v) == 'player101' \\\n RETURN v;\nTo match multiple VIDs, use WHERE id(v) IN [vid_list] or WHERE id(v) IN {vid_list}.\n MATCH (v:player { name: 'Tim Duncan' })--(v2) \\\n WHERE id(v2) IN [\"player101\", \"player102\"] \\\n RETURN v2;\n MATCH (v) WHERE id(v) IN {\"player100\", \"player101\"} \\\n RETURN v.player.name AS name;\nMatch connected vertices\nYou can use the -- symbol to represent edges of both directions and match vertices connected by these edges.\n MATCH (v:player{name:\"Tim Duncan\"})--(v2) \\\n RETURN v2.player.name AS Name;\n...\nYou can add a > or < to the -- symbol to specify the direction of an edge.\nIn the following example, --> represents an edge that starts from v and points to v2. To v, this is an outgoing edge, and to v2 this is an incoming edge.\n MATCH (v:player{name:\"Tim Duncan\"})-->(v2:player) \\\n RETURN v2.player.name AS Name;\nTo query the properties of the target vertices, use the CASE expression.\n MATCH (v:player{name:\"Tim Duncan\"})--(v2) \\\n RETURN \\\n CASE WHEN v2.team.name IS NOT NULL \\\n THEN v2.team.name \\\n WHEN v2.player.name IS NOT NULL \\\n THEN v2.player.name END AS Name;\n...\nTo extend the pattern, you can add more vertices and edges.\n MATCH (v:player{name:\"Tim Duncan\"})-->(v2)<--(v3) \\\n RETURN v3.player.name AS Name;\n...\nIf you do not need to refer to a vertex, you can omit the variable representing it in the parentheses.\n MATCH (v:player{name:\"Tim Duncan\"})-->()<--(v3) \\\n RETURN v3.player.name AS Name;\n...\nMatch paths\nConnected vertices and edges form a path. You can use a user-defined variable to name a path as follows.\n MATCH p=(v:player{name:\"Tim Duncan\"})-->(v2) \\\n RETURN p;\nMatch edges\n MATCH ()<-[e]-() \\\n RETURN e \\\n LIMIT 3;\nMatch edge types\nJust like vertices, you can specify edge types with : in a pattern. For example: -[e:follow]-.\n MATCH ()-[e:follow]->() \\\n RETURN e;\n...\nMatch edge type properties\nYou can specify edge type properties with {: } in a pattern. For example: [e:follow{likeness:95}].\n MATCH (v:player{name:\"Tim Duncan\"})-[e:follow{degree:95}]->(v2) \\\n RETURN e;\nUse the WHERE clause to directly get all the edges with the edge property value 90.\n MATCH ()-[e]->() \\\n WITH e, properties(e) as props, keys(properties(e)) as kk \\\n WHERE [i in kk where props[i] == 90] \\\n RETURN e;\n...\nMatch multiple edge types\nThe | symbol can help matching multiple edge types. For example: [e:follow|:serve]. The English colon (:) before the first edge type cannot be omitted, but the English colon before the subsequent edge type can be omitted, such as [e:follow|serve].\n MATCH (v:player{name:\"Tim Duncan\"})-[e:follow|:serve]->(v2) \\\n RETURN e;\nMatch multiple edges\nYou can extend a pattern to match multiple edges in a path.\n MATCH (v:player{name:\"Tim Duncan\"})-[]->(v2)<-[e:serve]-(v3) \\\n RETURN v2, v3;\n...\nMatch fixed-length paths\nYou can use the :* pattern to match a fixed-length path. hop must be a non-negative integer.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow*2]->(v2) \\\n RETURN DISTINCT v2 AS Friends;\nIf hop is 0, the pattern will match the source vertex of the path.\n MATCH (v:player{name:\"Tim Duncan\"}) -[*0]-> (v2) \\\n RETURN v2;\nMatch variable-length paths\nYou can use the :*[minHop..maxHop] pattern to match variable-length paths.minHop and maxHop are optional and default to 1\nand infinity respectively.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow*]->(v2) \\\n RETURN v2 AS Friends;\n...\n MATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow*1..3]->(v2) \\\n RETURN v2 AS Friends;\n...\n MATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow*1..]->(v2) \\\n RETURN v2 AS Friends;\n...\nYou can use the DISTINCT keyword to aggregate duplicate results.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow*1..3]->(v2:player) \\\n RETURN DISTINCT v2 AS Friends, count(v2);\nIf minHop is 0, the pattern will match the source vertex of the path. Compared to the preceding statement, the following example uses 0 as the minHop. So in the following result set, \"Tim Duncan\" is counted one more time than it is in the preceding result set because it is the source vertex.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow*0..3]->(v2:player) \\\n RETURN DISTINCT v2 AS Friends, count(v2);\nMatch variable-length paths with multiple edge types\nYou can specify multiple edge types in a fixed-length or variable-length pattern. In this case, hop, minHop, and maxHop take effect on all edge types.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[e:follow|serve*2]->(v2) \\\n RETURN DISTINCT v2;\nMatch multiple patterns\nYou can separate multiple patterns with commas (,).\n CREATE TAG INDEX IF NOT EXISTS team_index ON team(name(20));\n REBUILD TAG INDEX team_index;\n MATCH (v1:player{name:\"Tim Duncan\"}), (v2:team{name:\"Spurs\"}) \\\n RETURN v1,v2;\nMatch shortest paths\nThe allShortestPaths function can be used to find all shortest paths between two vertices.\n MATCH p = allShortestPaths((a:player{name:\"Tim Duncan\"})-[e*..5]-(b:player{name:\"Tony Parker\"})) \\\n RETURN p;\nThe shortestPath function can be used to find a single shortest path between two vertices.\n MATCH p = shortestPath((a:player{name:\"Tim Duncan\"})-[e*..5]-(b:player{name:\"Tony Parker\"})) \\\n RETURN p;\nRetrieve with multiple match\nMultiple MATCH can be used when different patterns have different filtering criteria and return the rows that exactly match the pattern.\n MATCH (m)-[]->(n) WHERE id(m)==\"player100\" \\\n MATCH (n)-[]->(l) WHERE id(n)==\"player125\" \\\n RETURN id(m),id(n),id(l);\nRetrieve with optional match\nSee OPTIONAL MATCH.","url":"7.general-query-statements/2.match/","type":"doc"},{"title":"OPTIONAL MATCH","content":"The OPTIONAL MATCH clause is used to search for the pattern described in it. OPTIONAL MATCH matches patterns against your graph database, just like MATCH does. The difference is that if no matches are found, OPTIONAL MATCH will use a null for missing parts of the pattern.\nOpenCypher Compatibility\nThis topic applies to the openCypher syntax in nGQL only.\nLimitations\nThe WHERE clause cannot be used in an OPTIONAL MATCH clause.\nExample\nThe example of the use of OPTIONAL MATCH in the MATCH statement is as follows:\n MATCH (m)-[]->(n) WHERE id(m)==\"player100\" \\\n OPTIONAL MATCH (n)-[]->(l) \\\n RETURN id(m),id(n),id(l);\nUsing multiple MATCH instead of OPTIONAL MATCH returns rows that match the pattern exactly. The example is as follows:\n MATCH (m)-[]->(n) WHERE id(m)==\"player100\" \\\n MATCH (n)-[]->(l) \\\n RETURN id(m),id(n),id(l);","url":"7.general-query-statements/optional-match/","type":"doc"},{"title":"LOOKUP","content":"The LOOKUP statement traverses data based on indexes. You can use LOOKUP for the following purposes:\nSearch for the specific data based on conditions defined by the WHERE clause.\nList vertices with a tag: retrieve the VID of all vertices with a tag.\nList edges with an edge type: retrieve the source vertex IDs, destination vertex IDs, and ranks of all edges with an edge type.\nCount the number of vertices or edges with a tag or an edge type.\nOpenCypher compatibility\nThis topic applies to native nGQL only.\nPrecautions\nCorrect use of indexes can speed up queries, but indexes can dramatically reduce the write performance. The performance can be greatly reduced. DO NOT use indexes in production environments unless you are fully aware of their influences on your service.\nIf the specified property is not indexed when using the LOOKUP statement, NebulaGraph randomly selects one of the available indexes.\nFor example, the tag player has two properties, name and age. Both the tag player itself and the property name have indexes, but the property age has no indexes. When running LOOKUP ON player WHERE player.age == 36 YIELD player.name;, NebulaGraph randomly uses one of the indexes of the tag player and the property name. You can use the EXPLAIN statement to check the selected index.\nPrerequisites\nBefore using the LOOKUP statement, make sure that at least one index is created. If there are already related vertices, edges, or properties before an index is created, the user must rebuild the index after creating the index to make it valid.\nSyntax\nLOOKUP ON { | }\n[WHERE [AND ...]]\nYIELD [DISTINCT] [AS ];\n\n [AS ] [, [AS ] ...];\nWHERE : filters data with specified conditions. Both AND and OR are supported between different expressions. For more information, see WHERE.\nYIELD: Define the output to be returned. For details, see YIELD.\nDISTINCT: Aggregate the output results and return the de-duplicated result set.\nAS: Set an alias.\nLimitations of using WHERE in LOOKUP\nThe WHERE clause in a LOOKUP statement does not support the following operations:\n$- and $^.\nFilter rank().\nIn relational expressions, operators are not supported to have field names on both sides, such as tagName.prop1> tagName.prop2.\nNested AliasProp expressions in operation expressions and function expressions are not supported.\nThe XOR operation is not supported.\nString operations other than STARTS WITH are not supported.\nGraph patterns.\nRetrieve vertices\nThe following example returns vertices whose name is Tony Parker and the tag is player.\n CREATE TAG INDEX IF NOT EXISTS index_player ON player(name(30), age);\n REBUILD TAG INDEX index_player;\n LOOKUP ON player \\\n WHERE player.name == \"Tony Parker\" \\\n YIELD id(vertex);\n LOOKUP ON player \\\n WHERE player.name == \"Tony Parker\" \\\n YIELD properties(vertex).name AS name, properties(vertex).age AS age;\n LOOKUP ON player \\\n WHERE player.age > 45 \\\n YIELD id(vertex);\n LOOKUP ON player \\\n WHERE player.name STARTS WITH \"B\" \\\n AND player.age IN [22,30] \\\n YIELD properties(vertex).name, properties(vertex).age;\n LOOKUP ON player \\\n WHERE player.name == \"Kobe Bryant\"\\\n YIELD id(vertex) AS VertexID, properties(vertex).name AS name |\\\n GO FROM $-.VertexID OVER serve \\\n YIELD $-.name, properties(edge).start_year, properties(edge).end_year, properties($$).name;\nRetrieve edges\nThe following example returns edges whose degree is 90 and the edge type is follow.\n CREATE EDGE INDEX IF NOT EXISTS index_follow ON follow(degree);\n REBUILD EDGE INDEX index_follow;\n LOOKUP ON follow \\\n WHERE follow.degree == 90 YIELD edge AS e;\n...\n LOOKUP ON follow \\\n WHERE follow.degree == 90 \\\n YIELD properties(edge).degree;\n...\n LOOKUP ON follow \\\n WHERE follow.degree == 60 \\\n YIELD dst(edge) AS DstVID, properties(edge).degree AS Degree |\\\n GO FROM $-.DstVID OVER serve \\\n YIELD $-.DstVID, properties(edge).start_year, properties(edge).end_year, properties($$).name;\nList vertices or edges with a tag or an edge type\nTo list vertices or edges with a tag or an edge type, at least one index must exist on the tag, the edge type, or its property.\nFor example, if there is a player tag with a name property and an age property, to retrieve the VID of all vertices tagged with player, there has to be an index on the player tag itself, the name property, or the age property.\nThe following example shows how to retrieve the VID of all vertices tagged with player. CREATE TAG IF NOT EXISTS player(name string,age int);\n CREATE TAG INDEX IF NOT EXISTS player_index on player();\n REBUILD TAG INDEX player_index;\n INSERT VERTEX player(name,age) \\\n VALUES \"player100\":(\"Tim Duncan\", 42), \"player101\":(\"Tony Parker\", 36);\nThe following statement retrieves the VID of all vertices with the tag `player`. It is similar to `MATCH (n:player) RETURN id(n) /*, n */`.\n LOOKUP ON player YIELD id(vertex);\n...\nThe following example shows how to retrieve the source Vertex IDs, destination vertex IDs, and ranks of all edges of the follow edge type. CREATE EDGE IF NOT EXISTS follow(degree int);\n CREATE EDGE INDEX IF NOT EXISTS follow_index on follow();\n REBUILD EDGE INDEX follow_index;\n INSERT EDGE follow(degree) \\\n VALUES \"player100\"->\"player101\":(95);\nThe following statement retrieves all edges with the edge type `follow`. It is similar to `MATCH (s)-[e:follow]->(d) RETURN id(s), rank(e), id(d) /*, type(e) */`.\nnebula)> LOOKUP ON follow YIELD edge AS e;\n...\nCount the numbers of vertices or edges\nThe following example shows how to count the number of vertices tagged with player and edges of the follow edge type.\n LOOKUP ON player YIELD id(vertex)|\\\n YIELD COUNT(*) AS Player_Number;\n LOOKUP ON follow YIELD edge AS e| \\\n YIELD COUNT(*) AS Follow_Number;","url":"7.general-query-statements/5.lookup/","type":"doc"},{"title":"GO","content":"The GO statement is used in the NebulaGraph database to traverse the graph starting from a given starting vertex with specified filters and return results.\nOpenCypher compatibility\nThis topic applies to native nGQL only.\nSyntax\nGO [[ TO] {STEP|STEPS}] FROM \nOVER [{REVERSELY | BIDIRECT}]\n[ WHERE  ]\nYIELD [DISTINCT] \n[{SAMPLE | }]\n[| GROUP BY {col_name | expr | position} YIELD ]\n[| ORDER BY [{ASC | DESC}]]\n[| LIMIT [,] ];\n ::=\n [, ...]\n ::=\n edge_type [, edge_type ...]\n | *\n ::=\n [AS ] [, [AS ] ...]\n {STEP|STEPS}: specifies the hop number. If not specified, the default value for N is one. When N is zero, NebulaGraph does not traverse any edges and returns nothing.\nM TO N {STEP|STEPS}: traverses from M to N hops. When M is zero, the output is the same as that of M is one. That is, the output of GO 0 TO 2 and GO 1 TO 2 are the same.\n: represents a list of vertex IDs separated by commas.\n: represents a list of edge types which the traversal can go through.\nREVERSELY | BIDIRECT: defines the direction of the query. By default, the GO statement searches for outgoing edges of . If REVERSELY is set, GO searches for incoming edges. If BIDIRECT is set, GO searches for edges of both directions. The direction of the query can be checked by returning the ._type field using YIELD. A positive value indicates an outgoing edge, while a negative value indicates an incoming edge.\nWHERE : specifies the traversal filters. You can use the WHERE clause for the source vertices, the edges, and the destination vertices. You can use it together with AND, OR, NOT, and XOR. For more information, see WHERE.\nYIELD [DISTINCT] : defines the output to be returned. It is recommended to use the Schema-related functions to fill in . src(edge), dst(edge), type(edge) ), rank(edge), etc., are currently supported, while nested functions are not. For more information, see YIELD.\nSAMPLE : takes samples from the result set. For more information, see SAMPLE.\n: limits the number of outputs during the traversal process. For more information, see LIMIT.\nGROUP BY: groups the output into subgroups based on the value of the specified property. For more information, see GROUP BY. After grouping, you need to use YIELD again to define the output that needs to be returned.\nORDER BY: sorts outputs with specified orders. For more information, see ORDER BY.\nLIMIT [,] ]: limits the number of rows of the output. For more information, see LIMIT.\nNotes\nThe WHERE and YIELD clauses in GO statements usually utilize property reference symbols ($^ and $$) or the properties($^) and properties($$) functions to specify the properties of a vertex; use the properties(edge) function to specify the properties of an edge. For details, see Property Reference Symbols and Schema-related Functions.\nWhen referring to the result of a subquery in a compound GO statement, you need to set a name for the result and pass it to the next subquery using the pipe symbol |, and reference the name of the result in the next subquery using $-. See the Pipe Operator for details.\nWhen the queried property has no value, the returned result displays NULL.\nCases and examples\nTo query the immediate neighbors of a vertex\nFor example, to query the team that a person belongs to, assuming that the person is connected to the team by the serve edge and the person's ID is player102.\n GO FROM \"player102\" OVER serve YIELD dst(edge);\nTo query all vertices within a specified number of hops from a starting vertex\nFor example, to query all vertices within two hops of a person vertex, assuming that the person is connected to other people by the follow edge and the person's ID is player102.\n# Return all vertices that are 2 hops away from the player102 vertex.\n GO 2 STEPS FROM \"player102\" OVER follow YIELD dst(edge);\n# Return all vertices within 1 or 2 hops away from the player102 vertex.\n GO 1 TO 2 STEPS FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS destination;\n...\n# The following MATCH query has the same semantics as the previous GO query.\n MATCH (v) -[e:follow*1..2]->(v2) \\\n WHERE id(v) == \"player100\" \\\n RETURN id(v2) AS destination;\n...\nTo add filtering conditions\nCase: To query the vertices and edges that meet specific conditions.\nFor example, use the WHERE clause to query the edges with specific properties between the starting vertex and the destination vertex.\n GO FROM \"player100\", \"player102\" OVER serve \\\n WHERE properties(edge).start_year > 1995 \\\n YIELD DISTINCT properties($$).name AS team_name, properties(edge).start_year AS start_year, properties($^).name AS player_name;\nTo query all edges\nCase: To query all edges that are connected to the starting vertex.\n# Return all edges that are connected to the player102 vertex.\n GO FROM \"player102\" OVER * BIDIRECT YIELD edge AS e;\nTo query multiple edge types\nCase: To query multiple edge types that are connected to the starting vertex. You can specify multiple edge types or the * symbol to query multiple edge types.\nFor example, to query the follow and serve edges that are connected to the starting vertex.\n GO FROM \"player100\" OVER follow, serve \\\n YIELD properties(edge).degree, properties(edge).start_year;\nTo query incoming vertices using the REVERSELY keyword\n# Return the vertices that follow the player100 vertex.\n GO FROM \"player100\" OVER follow REVERSELY \\\n YIELD src(edge) AS destination;\n...\n# The following MATCH query has the same semantics as the previous GO query.\n MATCH (v)<-[e:follow]- (v2) WHERE id(v) == 'player100' \\\n RETURN id(v2) AS destination;\n...\nTo use subqueries as the starting vertice of a graph traversal\n# Return the friends of the player100 vertex and the teams that the friends belong to.\n GO FROM \"player100\" OVER follow REVERSELY \\\n YIELD src(edge) AS id | \\\n GO FROM $-.id OVER serve \\\n WHERE properties($^).age > 20 \\\n YIELD properties($^).name AS FriendOf, properties($$).name AS Team;\n...\n# The following MATCH query has the same semantics as the previous GO query.\n MATCH (v)<-[e:follow]- (v2)-[e2:serve]->(v3) \\\n WHERE id(v) == 'player100' \\\n RETURN v2.player.name AS FriendOf, v3.team.name AS Team;\n...\nTo use GROUP BY to group the output\nYou need to use YIELD to define the output that needs to be returned after grouping.\n# The following example collects the outputs according to age.\n GO 2 STEPS FROM \"player100\" OVER follow \\\n YIELD src(edge) AS src, dst(edge) AS dst, properties($$).age AS age \\\n | GROUP BY $-.dst \\\n YIELD $-.dst AS dst, collect_set($-.src) AS src, collect($-.age) AS age;\nTo use ORDER BY and LIMIT to sort and limit the output\n# The following example groups the outputs and restricts the number of rows of the outputs.\n $a = GO FROM \"player100\" OVER follow YIELD src(edge) AS src, dst(edge) AS dst; \\\n GO 2 STEPS FROM $a.dst OVER follow \\\n YIELD $a.src AS src, $a.dst, src(edge), dst(edge) \\\n | ORDER BY $-.src | OFFSET 1 LIMIT 2;\nOther examples\n# The following example determines if $$.player.name IS NOT EMPTY.\n GO FROM \"player100\" OVER follow WHERE properties($$).name IS NOT EMPTY YIELD dst(edge);","url":"7.general-query-statements/3.go/","type":"doc"},{"title":"FETCH","content":"The FETCH statement retrieves the properties of the specified vertices or edges.\nOpenCypher Compatibility\nThis topic applies to native nGQL only.\nFetch vertex properties\nSyntax\nFETCH PROP ON {[, tag_name ...] | *}\n [, vid ...]\nYIELD [DISTINCT] [AS ];\nParameter\nDescription\ntag_name\nThe name of the tag.\nRepresents all the tags in the current graph space.\nvid\nThe vertex ID.\nYIELD\nDefine the output to be returned. For details, see YIELD.\nAS\nSet an alias.\nFetch vertex properties by one tag\nSpecify a tag in the FETCH statement to fetch the vertex properties by that tag.\n FETCH PROP ON player \"player100\" YIELD properties(vertex);\nFetch specific properties of a vertex\nUse a YIELD clause to specify the properties to be returned.\n FETCH PROP ON player \"player100\" \\\n YIELD properties(vertex).name AS name;\nFetch properties of multiple vertices\nSpecify multiple VIDs (vertex IDs) to fetch properties of multiple vertices. Separate the VIDs with commas.\n FETCH PROP ON player \"player101\", \"player102\", \"player103\" YIELD properties(vertex);\nFetch vertex properties by multiple tags\nSpecify multiple tags in the FETCH statement to fetch the vertex properties by the tags. Separate the tags with commas.\n# The following example creates a new tag t1.\n CREATE TAG IF NOT EXISTS t1(a string, b int);\n# The following example attaches t1 to the vertex \"player100\".\n INSERT VERTEX t1(a, b) VALUES \"player100\":(\"Hello\", 100);\n# The following example fetches the properties of vertex \"player100\" by the tags player and t1.\n FETCH PROP ON player, t1 \"player100\" YIELD vertex AS v;\nYou can combine multiple tags with multiple VIDs in a FETCH statement.\n FETCH PROP ON player, t1 \"player100\", \"player103\" YIELD vertex AS v;\nFetch vertex properties by all tags\nSet an asterisk symbol * to fetch properties by all tags in the current graph space.\n FETCH PROP ON * \"player100\", \"player106\", \"team200\" YIELD vertex AS v;\nFetch edge properties\nSyntax\nFETCH PROP ON -> [@] [, -> ...]\nYIELD ;\nParameter\nDescription\nedge_type\nThe name of the edge type.\nsrc_vid\nThe VID of the source vertex. It specifies the start of an edge.\ndst_vid\nThe VID of the destination vertex. It specifies the end of an edge.\nrank\nThe rank of the edge. It is optional and defaults to 0. It distinguishes an edge from other edges with the same edge type, source vertex, destination vertex, and rank.\nYIELD\nDefine the output to be returned. For details, see YIELD.\nFetch all properties of an edge\nThe following statement fetches all the properties of the serve edge that connects vertex \"player100\" and vertex \"team204\".\n FETCH PROP ON serve \"player100\" -> \"team204\" YIELD properties(edge);\nFetch specific properties of an edge\nUse a YIELD clause to fetch specific properties of an edge.\n FETCH PROP ON serve \"player100\" -> \"team204\" \\\n YIELD properties(edge).start_year;\nFetch properties of multiple edges\nSpecify multiple edge patterns ( -> [@]) to fetch properties of multiple edges. Separate the edge patterns with commas.\n FETCH PROP ON serve \"player100\" -> \"team204\", \"player133\" -> \"team202\" YIELD edge AS e;\nFetch properties based on edge rank\nIf there are multiple edges with the same edge type, source vertex, and destination vertex, you can specify the rank to fetch the properties on the correct edge.\n# The following example inserts edges with different ranks and property values.\n insert edge serve(start_year,end_year) \\\n values \"player100\"->\"team204\"@1:(1998, 2017);\n insert edge serve(start_year,end_year) \\\n values \"player100\"->\"team204\"@2:(1990, 2018);\n# By default, the FETCH statement returns the edge whose rank is 0.\n FETCH PROP ON serve \"player100\" -> \"team204\" YIELD edge AS e;\n# To fetch on an edge whose rank is not 0, set its rank in the FETCH statement.\n FETCH PROP ON serve \"player100\" -> \"team204\"@1 YIELD edge AS e;\nUse FETCH in composite queries\nA common way to use FETCH is to combine it with native nGQL such as GO.\nThe following statement returns the degree values of the follow edges that start from vertex \"player101\".\n GO FROM \"player101\" OVER follow \\\n YIELD src(edge) AS s, dst(edge) AS d \\\n | FETCH PROP ON follow $-.s -> $-.d \\\n YIELD properties(edge).degree;\nOr you can use user-defined variables to construct similar queries.\n $var = GO FROM \"player101\" OVER follow \\\n YIELD src(edge) AS s, dst(edge) AS d; \\\n FETCH PROP ON follow $var.s -> $var.d \\\n YIELD properties(edge).degree;\nFor more information about composite queries, see Composite queries (clause structure).","url":"7.general-query-statements/4.fetch/","type":"doc"},{"title":"SHOW CHARSET","content":"The SHOW CHARSET statement shows the available character sets.\nCurrently available types are utf8 and utf8mb4. The default charset type is utf8. NebulaGraph extends the uft8 to support four-byte characters. Therefore utf8 and utf8mb4 are equivalent.\nSyntax\nSHOW CHARSET;\nExample\n SHOW CHARSET;\nParameter\nDescription\nCharset\nThe name of the character set.\nDescription\nThe description of the character set.\nDefault collation\nThe default collation of the character set.\nMaxlen\nThe maximum number of bytes required to store one character.","url":"7.general-query-statements/6.show/1.show-charset/","type":"doc"},{"title":"SHOW COLLATION","content":"The SHOW COLLATION statement shows the collations supported by NebulaGraph.\nCurrently available types are: utf8_bin and utf8mb4_bin.\nWhen the character set is utf8, the default collate is utf8_bin.\nWhen the character set is utf8mb4, the default collate is utf8mb4_bin.\nSyntax\nSHOW COLLATION;\nExample\n SHOW COLLATION;\nParameter\nDescription\nCollation\nThe name of the collation.\nCharset\nThe name of the character set with which the collation is associated.","url":"7.general-query-statements/6.show/2.show-collation/","type":"doc"},{"title":"SHOW CREATE SPACE","content":"The SHOW CREATE SPACE statement shows the creating statement of the specified graph space.\nFor details about the graph space information, see CREATE SPACE.\nSyntax\nSHOW CREATE SPACE ;\nExample\n SHOW CREATE SPACE basketballplayer;","url":"7.general-query-statements/6.show/4.show-create-space/","type":"doc"},{"title":"SHOW CREATE TAG/EDGE","content":"The SHOW CREATE TAG statement shows the basic information of the specified tag. For details about the tag, see CREATE TAG.\nThe SHOW CREATE EDGE statement shows the basic information of the specified edge type. For details about the edge type, see CREATE EDGE.\nSyntax\nSHOW CREATE {TAG | EDGE };\nExamples\n SHOW CREATE TAG player;\n SHOW CREATE EDGE follow;","url":"7.general-query-statements/6.show/5.show-create-tag-edge/","type":"doc"},{"title":"SHOW HOSTS","content":"The SHOW HOSTS statement shows the cluster information, including the port, status, leader, partition, and version information. You can also add the service type in the statement to view the information of the specific service.\nSyntax\nSHOW HOSTS [GRAPH | STORAGE | META];\nExamples\n SHOW HOSTS;\n SHOW HOSTS GRAPH;\n SHOW HOSTS STORAGE;\n SHOW HOSTS META;","url":"7.general-query-statements/6.show/6.show-hosts/","type":"doc"},{"title":"SHOW INDEX STATUS","content":"The SHOW INDEX STATUS statement shows the status of jobs that rebuild native indexes, which helps check whether a native index is successfully rebuilt or not.\nSyntax\nSHOW {TAG | EDGE} INDEX STATUS;\nExamples\n SHOW TAG INDEX STATUS;\n SHOW EDGE INDEX STATUS;\nRelated topics\nJob manager and the JOB statements\nREBUILD NATIVE INDEX","url":"7.general-query-statements/6.show/7.show-index-status/","type":"doc"},{"title":"SHOW INDEXES","content":"The SHOW INDEXES statement shows the names of existing native indexes.\nSyntax\nSHOW {TAG | EDGE} INDEXES;\nExamples\n SHOW TAG INDEXES;\n SHOW EDGE INDEXES;","url":"7.general-query-statements/6.show/8.show-indexes/","type":"doc"},{"title":"SHOW PARTS","content":"The SHOW PARTS statement shows the information of a specified partition or all partitions in a graph space.\nSyntax\nSHOW PARTS [];\nExamples\n SHOW PARTS;\n SHOW PARTS 1;\nThe descriptions are as follows.\nParameter\nDescription\nPartition ID\nThe ID of the partition.\nLeader\nThe IP (or hostname) and the port of the leader.\nPeers\nThe IPs (or hostnames) and the ports of all the replicas.\nLosts\nThe IPs (or hostnames) and the ports of replicas at fault.","url":"7.general-query-statements/6.show/9.show-parts/","type":"doc"},{"title":"SHOW ROLES","content":"The SHOW ROLES statement shows the roles that are assigned to a user account.\nThe return message differs according to the role of the user who is running this statement:\nIf the user is a GOD or ADMIN and is granted access to the specified graph space, NebulaGraph shows all roles in this graph space except for GOD.\nIf the user is a DBA, USER, or GUEST and is granted access to the specified graph space, NebulaGraph shows the user's own role in this graph space.\nIf the user does not have access to the specified graph space, NebulaGraph returns PermissionError.\nFor more information about roles, see Roles and privileges.\nSyntax\nSHOW ROLES IN ;\nExample\n SHOW ROLES in basketballplayer;","url":"7.general-query-statements/6.show/10.show-roles/","type":"doc"},{"title":"SHOW SNAPSHOTS","content":"The SHOW SNAPSHOTS statement shows the information of all the snapshots.\nFor how to create a snapshot and backup data, see Snapshot.\nRole requirement\nOnly the root user who has the GOD role can use the SHOW SNAPSHOTS statement.\nSyntax\nSHOW SNAPSHOTS;\nExample\n SHOW SNAPSHOTS;","url":"7.general-query-statements/6.show/11.show-snapshots/","type":"doc"},{"title":"SHOW SPACES","content":"The SHOW SPACES statement shows existing graph spaces in NebulaGraph.\nFor how to create a graph space, see CREATE SPACE.\nSyntax\nSHOW SPACES;\nExample\n SHOW SPACES;","url":"7.general-query-statements/6.show/12.show-spaces/","type":"doc"},{"title":"SHOW STATS","content":"The SHOW STATS statement shows the statistics of the graph space collected by the latest SUBMIT JOB STATS job.\nThe statistics include the following information:\nThe number of vertices in the graph space\nThe number of edges in the graph space\nThe number of vertices of each tag\nThe number of edges of each edge type\nPrerequisites\nYou have to run the SUBMIT JOB STATS statement in the graph space where you want to collect statistics. For more information, see SUBMIT JOB STATS.\nSyntax\nSHOW STATS;\nExamples\n# Choose a graph space.\n USE basketballplayer;\n# Start SUBMIT JOB STATS.\n SUBMIT JOB STATS;\n# Make sure the job executes successfully.\n SHOW JOB 98;\n# Show the statistics of the graph space.\n SHOW STATS;","url":"7.general-query-statements/6.show/14.show-stats/","type":"doc"},{"title":"SHOW TAGS/EDGES","content":"The SHOW TAGS statement shows all the tags in the current graph space.\nThe SHOW EDGES statement shows all the edge types in the current graph space.\nSyntax\nSHOW {TAGS | EDGES};\nExamples\n SHOW TAGS;\n SHOW EDGES;","url":"7.general-query-statements/6.show/15.show-tags-edges/","type":"doc"},{"title":"SHOW USERS","content":"The SHOW USERS statement shows the user information.\nRole requirement\nOnly the root user who has the GOD role can use the SHOW USERS statement.\nSyntax\nSHOW USERS;\nExample\n SHOW USERS;","url":"7.general-query-statements/6.show/16.show-users/","type":"doc"},{"title":"SHOW SESSIONS","content":"When a user logs in to the database, a corresponding session will be created and users can query for session information.\nThe SHOW SESSIONS statement shows the information of all the sessions. It can also show a specified session with its ID.\nPrecautions\nThe client will call the API release to release the session and clear the session information when you run exit after the operation ends. If you exit the database in an unexpected way and the session timeout duration is not set via session_idle_timeout_secs in nebula-graphd.conf, the session will not be released automatically. For those sessions that are not automatically released, you need to delete them manually. For details, see KILL SESSIONS.\nSHOW SESSIONS queries the session information of all the Graph services.\nSHOW LOCAL SESSIONS queries the session information of the currently connected Graph service and does not query the session information of other Graph services.\nSHOW SESSION queries the session information with a specific session id.\nSyntax\nSHOW [LOCAL] SESSIONS;\nSHOW SESSION ;\nExamples\n SHOW SESSIONS;\n SHOW SESSION 1635254859271703;\nParameter\nDescription\nSessionId\nThe session ID, namely the identifier of a session.\nUserName\nThe username in a session.\nSpaceName\nThe name of the graph space that the user uses currently. It is null (\"\") when you first log in because there is no specified graph space.\nCreateTime\nThe time when the session is created, namely the time when the user logs in. The time zone is specified by timezone_name in the configuration file.\nUpdateTime\nThe system will update the time when there is an operation. The time zone is specified by timezone_name in the configuration file.\nGraphAddr\nThe IP (or hostname) and port of the Graph server that hosts the session.\nTimezone\nA reserved parameter that has no specified meaning for now.\nClientIp\nThe IP or hostname of the client.","url":"7.general-query-statements/6.show/17.show-sessions/","type":"doc"},{"title":"SHOW QUERIES","content":"The SHOW QUERIES statement shows the information of working queries in the current session.\nPrecautions\nThe SHOW LOCAL QUERIES statement gets the status of queries in the current session from the local cache with almost no latency.\nThe SHOW QUERIES statement gets the information of queries in all the sessions from the Meta Service. The information will be synchronized to the Meta Service according to the interval defined by session_reclaim_interval_secs. Therefore the information that you get from the client may belong to the last synchronization interval.\nSyntax\nSHOW [LOCAL] QUERIES;\nExamples\n SHOW LOCAL QUERIES;\n SHOW QUERIES;\n# The following statement returns the top 10 queries that have the longest duration.\n SHOW QUERIES | ORDER BY $-.DurationInUSec DESC | LIMIT 10;\nThe descriptions are as follows.\nParameter\nDescription\nSessionID\nThe session ID.\nExecutionPlanID\nThe ID of the execution plan.\nUser\nThe username that executes the query.\nHost\nThe IP address and port of the Graph server that hosts the session.\nStartTime\nThe time when the query starts.\nDurationInUSec\nThe duration of the query. The unit is microsecond.\nStatus\nThe current status of the query.\nQuery\nThe query statement.","url":"7.general-query-statements/6.show/18.show-queries/","type":"doc"},{"title":"SHOW META LEADER","content":"The SHOW META LEADER statement shows the information of the leader in the current Meta cluster.\nFor more information about the Meta service, see Meta service.\nSyntax\nSHOW META LEADER;\nExample\n SHOW META LEADER;\nParameter\nDescription\nMeta Leader\nShows the information of the leader in the Meta cluster, including the IP (or hostname) and port of the server where the leader is located.\nsecs from last heart beat\nIndicates the time interval since the last heartbeat. This parameter is measured in seconds.","url":"7.general-query-statements/6.show/19.show-meta-leader/","type":"doc"},{"title":"FIND PATH","content":"The FIND PATH statement finds the paths between the selected source vertices and destination vertices.\nSyntax\nFIND { SHORTEST | SINGLE SHORTEST | ALL | NOLOOP } PATH [WITH PROP] FROM TO \nOVER [REVERSELY | BIDIRECT] \n[] [UPTO {STEP|STEPS}] \nYIELD path as \n[| ORDER BY $-.path] [| LIMIT ];\n ::=\n [vertex_id [, vertex_id] ...]\nSHORTEST finds all the shortest path.\nALL finds all the paths.\nNOLOOP finds the paths without circles.\nWITH PROP shows properties of vertices and edges. If not specified, properties will be hidden.\n is a list of vertex IDs separated with commas (,). It supports $- and $var.\n is a list of edge types separated with commas (,). * is all edge types.\nREVERSELY | BIDIRECT specifies the direction. REVERSELY is reverse graph traversal while BIDIRECT is bidirectional graph traversal.\n filters properties of edges.\nUPTO {STEP|STEPS} is the maximum hop number of the path. The default value is 5.\nORDER BY $-.path specifies the order of the returned paths. For information about the order rules, see Path.\nLIMIT specifies the maximum number of rows to return.\nLimitations\nWhen a list of source and/or destination vertex IDs are specified, the paths between any source vertices and the destination vertices will be returned.\nThere can be cycles when searching all paths.\nFIND PATH only supports filtering properties of edges with WHERE clauses. Filtering properties of vertices and functions are not supported for now.\nFIND PATH is a single-thread procedure, so it uses much memory.\nExamples\nA returned path is like ()-[:@]->(=0 YIELD path AS p;\n FIND NOLOOP PATH FROM \"player100\" TO \"team204\" OVER * YIELD path AS p;\nFAQ\nDoes it support the WHERE clause to achieve conditional filtering during graph traversal?\nFIND PATH only supports filtering properties of edges with WHERE clauses, such as WHERE follow.degree is EMPTY or follow.degree >=0.\nFiltering properties of vertices is not supported for now.","url":"7.general-query-statements/6.find-path/","type":"doc"},{"title":"GET SUBGRAPH","content":"The GET SUBGRAPH statement returns a subgraph that is generated by traversing a graph starting from a specified vertex. GET SUBGRAPH statements allow you to specify the number of steps and the type or direction of edges during the traversal.\nSyntax\nGET SUBGRAPH [WITH PROP] [ {STEP|STEPS}] FROM {, ...}\n[{IN | OUT | BOTH} , ...]\n[WHERE [AND ...]]\nYIELD {[VERTICES AS ] [,EDGES AS ]};\nWITH PROP shows the properties. If not specified, the properties will be hidden.\nstep_count specifies the number of hops from the source vertices and returns the subgraph from 0 to step_count hops. It must be a non-negative integer. Its default value is 1.\nvid specifies the vertex IDs. \nedge_type specifies the edge type. You can use IN, OUT, and BOTH to specify the traversal direction of the edge type. The default is BOTH.\n specifies the filter conditions for the traversal, which can be used with the boolean operator AND.\nYIELD defines the output that needs to be returned. You can return only vertices or edges. A column alias must be set.\nLimitations\nWhile using the WHERE clause in a GET SUBGRAPH statement, note the following restrictions:\nOnly support the AND operator.\nOnly support filter destination vertex, the vertex format must be $$.tagName.propName.\nSupport filter edge, the edge format must be edge_type.propName.\nSupport math functions, aggregate functions, string functions, datetime functions, type conversion functions and general functions in list functions.\nNot support aggregate functions, schema-related functions, conditional expression, predicate functions, geography function and user-defined functions.\nExamples\nThe following graph is used as the sample.\nInsert the test data:\n CREATE SPACE IF NOT EXISTS subgraph(partition_num=15, replica_factor=1, vid_type=fixed_string(30));\n USE subgraph;\n CREATE TAG IF NOT EXISTS player(name string, age int);\n CREATE TAG IF NOT EXISTS team(name string);\n CREATE EDGE IF NOT EXISTS follow(degree int);\n CREATE EDGE IF NOT EXISTS serve(start_year int, end_year int);\n INSERT VERTEX player(name, age) VALUES \"player100\":(\"Tim Duncan\", 42);\n INSERT VERTEX player(name, age) VALUES \"player101\":(\"Tony Parker\", 36);\n INSERT VERTEX player(name, age) VALUES \"player102\":(\"LaMarcus Aldridge\", 33);\n INSERT VERTEX team(name) VALUES \"team203\":(\"Trail Blazers\"), \"team204\":(\"Spurs\");\n INSERT EDGE follow(degree) VALUES \"player101\" -> \"player100\":(95);\n INSERT EDGE follow(degree) VALUES \"player101\" -> \"player102\":(90);\n INSERT EDGE follow(degree) VALUES \"player102\" -> \"player100\":(75);\n INSERT EDGE serve(start_year, end_year) VALUES \"player101\" -> \"team204\":(1999, 2018),\"player102\" -> \"team203\":(2006, 2015);\nThis example goes one step from the vertex player101 over all edge types and gets the subgraph. GET SUBGRAPH 1 STEPS FROM \"player101\" YIELD VERTICES AS nodes, EDGES AS relationships;\nThe returned subgraph is as follows.\nThis example goes one step from the vertex player101 over incoming follow edges and gets the subgraph. GET SUBGRAPH 1 STEPS FROM \"player101\" IN follow YIELD VERTICES AS nodes, EDGES AS relationships;\nThere is no incoming follow edge to player101, so only the vertex player101 is returned.\nThis example goes one step from the vertex player101 over outgoing serve edges, gets the subgraph, and shows the property of the edge. GET SUBGRAPH WITH PROP 1 STEPS FROM \"player101\" OUT serve YIELD VERTICES AS nodes, EDGES AS relationships;\nThe returned subgraph is as follows.\nThis example goes two steps from the vertex player101 over follow edges, filters by degree > 90 and age > 30, and shows the properties of edges. GET SUBGRAPH WITH PROP 2 STEPS FROM \"player101\" \\\n WHERE follow.degree > 90 AND $$.player.age > 30 \\\n YIELD VERTICES AS nodes, EDGES AS relationships;\nFAQ\nWhy is the number of hops in the returned result greater than step_count?\nTo show the completeness of the subgraph, an additional hop is made on all vertices that meet the conditions. The following graph is used as the sample.\nThe returned paths of GET SUBGRAPH 1 STEPS FROM \"A\"; are A->B, B->A, and A->C. To show the completeness of the subgraph, an additional hop is made on all vertices that meet the conditions, namely B->C.\nThe returned path of GET SUBGRAPH 1 STEPS FROM \"A\" IN follow; is B->A. To show the completeness of the subgraph, an additional hop is made on all vertices that meet the conditions, namely A->B.\nIf you only query paths or vertices that meet the conditions, we suggest you use MATCH or GO. The example is as follows.\n MATCH p= (v:player) -- (v2) WHERE id(v)==\"A\" RETURN p;\n GO 1 STEPS FROM \"A\" OVER follow YIELD src(edge),dst(edge);\nWhy is the number of hops in the returned result lower than step_count?\nThe query stops when there is not enough subgraph data and will not return the null value.\n GET SUBGRAPH 100 STEPS FROM \"player101\" OUT follow YIELD VERTICES AS nodes, EDGES AS relationships;","url":"7.general-query-statements/7.get-subgraph/","type":"doc"},{"title":"GROUP BY","content":"The GROUP BY clause can be used to aggregate data.\nOpenCypher Compatibility\nThis topic applies to native nGQL only.\nYou can also use the count() function to aggregate data.\n MATCH (v:player)<-[:follow]-(:player) RETURN v.player.name AS Name, count(*) as cnt ORDER BY cnt DESC;\n...\nSyntax\nThe GROUP BY clause groups the rows with the same value. Then operations such as counting, sorting, and calculation can be applied.\nThe GROUP BY clause works after the pipe symbol (|) and before a YIELD clause.\n| GROUP BY YIELD , \nThe aggregation_function() function supports avg(), sum(), max(), min(), count(), collect(), and std().\nExamples\nThe following statement finds all the vertices connected directly to vertex \"player100\", groups the result set by player names, and counts how many times the name shows up in the result set.\n GO FROM \"player100\" OVER follow BIDIRECT \\\n YIELD properties($$).name as Name \\\n | GROUP BY $-.Name \\\n YIELD $-.Name as Player, count(*) AS Name_Count;\nThe following statement finds all the vertices connected directly to vertex \"player100\", groups the result set by source vertices, and returns the sum of degree values.\n GO FROM \"player100\" OVER follow \\\n YIELD src(edge) AS player, properties(edge).degree AS degree \\\n | GROUP BY $-.player \\\n YIELD sum($-.degree);\nFor more information about the sum() function, see Built-in math functions.\nImplicit GROUP BY\nThe usage of GROUP BY in the above nGQL statements that explicitly write GROUP BY and act as grouping fields is called explicit GROUP BY, while in openCypher, the GROUP BY is implicit, i.e., GROUP BY groups fields without explicitly writing GROUP BY. The explicit GROUP BY in nGQL is the same as the implicit GROUP BY in openCypher, and nGQL also supports the implicit GROUP BY. For the implicit usage of GROUP BY, see Stack Overflow.\nFor example, to look up the players over 34 years old with the same length of service, you can use the following statement:\n LOOKUP ON player WHERE player.age > 34 YIELD id(vertex) AS v | \\\n GO FROM $-.v OVER serve YIELD serve.start_year AS start_year, serve.end_year AS end_year | \\\n YIELD $-.start_year, $-.end_year, count(*) AS count | \\\n ORDER BY $-.count DESC | LIMIT 5;\n+---------------+-------------+-------+ ","url":"8.clauses-and-options/group-by/","type":"doc"},{"title":"LIMIT AND SKIP","content":"The LIMIT clause constrains the number of rows in the output. The usage of LIMIT in native nGQL statements and openCypher compatible statements is different.\nNative nGQL: Generally, a pipe | needs to be used before the LIMIT clause. The offset parameter can be set or omitted directly after the LIMIT statement.\nOpenCypher compatible statements: No pipes are permitted before the LIMIT clause. And you can use SKIP to indicate an offset.\nLIMIT in native nGQL statements\nIn native nGQL, LIMIT has general syntax and exclusive syntax in GO statements.\nGeneral LIMIT syntax in native nGQL statements\nIn native nGQL, the general LIMIT syntax works the same as in SQL. The LIMIT clause accepts one or two parameters. The values of both parameters must be non-negative integers and be used after a pipe. The syntax and description are as follows:\n... | LIMIT [,] ;\nParameter\nDescription\noffset\nThe offset value. It defines the row from which to start returning. The offset starts from 0. The default value is 0, which returns from the first row.\nnumber_rows\nIt constrains the total number of returned rows.\nFor example:\n# The following example returns the top 3 rows of data from the result.\n LOOKUP ON player YIELD id(vertex)|\\\n LIMIT 3;\n# The following example returns the 3 rows of data starting from the second row of the sorted output.\n GO FROM \"player100\" OVER follow REVERSELY \\\n YIELD properties($$).name AS Friend, properties($$).age AS Age \\\n | ORDER BY $-.Age, $-.Friend \\\n | LIMIT 1, 3;\nLIMIT in GO statements\nIn addition to the general syntax in the native nGQL, the LIMIT in the GO statement also supports limiting the number of output results based on edges.\nSyntax:\n LIMIT ;\nlimit_list is a list. Elements in the list must be natural numbers, and the number of elements must be the same as the maximum number of STEPS in the GO statement. The following takes GO 1 TO 3 STEPS FROM \"A\" OVER * LIMIT as an example to introduce this usage of LIMIT in detail.\nThe list limit_list must contain 3 natural numbers, such as GO 1 TO 3 STEPS FROM \"A\" OVER * LIMIT [1,2,4].\n1 in LIMIT [1,2,4] means that the system automatically selects 1 edge to continue traversal in the first step. 2 means to select 2 edges to continue traversal in the second step. 4 indicates that 4 edges are selected to continue traversal in the third step.\nBecause GO 1 TO 3 STEPS means to return all the traversal results from the first to third steps, all the red edges and their source and destination vertices in the figure below will be matched by this GO statement. And the yellow edges represent there is no path selected when the GO statement traverses. If it is not GO 1 TO 3 STEPS but GO 3 STEPS, it will only match the red edges of the third step and the vertices at both ends.\nIn the basketballplayer dataset, the example is as follows:\n GO 3 STEPS FROM \"player100\" \\\n OVER * \\\n YIELD properties($$).name AS NAME, properties($$).age AS Age \\\n LIMIT [3,3,3];\n GO 3 STEPS FROM \"player102\" OVER * BIDIRECT\\\n YIELD dst(edge) \\\n LIMIT [rand32(5),rand32(5),rand32(5)];\nLIMIT in openCypher compatible statements\nIn openCypher compatible statements such as MATCH, there is no need to use a pipe when LIMIT is used. The syntax and description are as follows:\n... [SKIP ] [LIMIT ];\nParameter\nDescription\noffset\nThe offset value. It defines the row from which to start returning. The offset starts from 0. The default value is 0, which returns from the first row.\nnumber_rows\nIt constrains the total number of returned rows.\nBoth offset and number_rows accept expressions, but the result of the expression must be a non-negative integer.\nExamples of LIMIT\nLIMIT can be used alone to return a specified number of results.\n MATCH (v:player) RETURN v.player.name AS Name, v.player.age AS Age \\\n ORDER BY Age LIMIT 5;\nExamples of SKIP\nSKIP can be used alone to set the offset and return the data after the specified position.\n MATCH (v:player{name:\"Tim Duncan\"}) --> (v2) \\\n RETURN v2.player.name AS Name, v2.player.age AS Age \\\n ORDER BY Age DESC SKIP 1;\n MATCH (v:player{name:\"Tim Duncan\"}) --> (v2) \\\n RETURN v2.player.name AS Name, v2.player.age AS Age \\\n ORDER BY Age DESC SKIP 1+1;\nExample of SKIP and LIMIT\nSKIP and LIMIT can be used together to return the specified amount of data starting from the specified position.\n MATCH (v:player{name:\"Tim Duncan\"}) --> (v2) \\\n RETURN v2.player.name AS Name, v2.player.age AS Age \\\n ORDER BY Age DESC SKIP 1 LIMIT 1;","url":"8.clauses-and-options/limit/","type":"doc"},{"title":"SAMPLE","content":"The SAMPLE clause takes samples evenly in the result set and returns the specified amount of data.\nSAMPLE can be used in GO statements only. The syntax is as follows:\n SAMPLE ;\nsample_list is a list. Elements in the list must be natural numbers, and the number of elements must be the same as the maximum number of STEPS in the GO statement. The following takes GO 1 TO 3 STEPS FROM \"A\" OVER * SAMPLE as an example to introduce this usage of SAMPLE in detail.\nThe list sample_list must contain 3 natural numbers, such as GO 1 TO 3 STEPS FROM \"A\" OVER * SAMPLE [1,2,4].\n1 in SAMPLE [1,2,4] means that the system automatically selects 1 edge to continue traversal in the first step. 2 means to select 2 edges to continue traversal in the second step. 4 indicates that 4 edges are selected to continue traversal in the third step. If there is no matched edge in a certain step or the number of matched edges is less than the specified number, the actual number will be returned.\nBecause GO 1 TO 3 STEPS means to return all the traversal results from the first to third steps, all the red edges and their source and destination vertices in the figure below will be matched by this GO statement. And the yellow edges represent there is no path selected when the GO statement traverses. If it is not GO 1 TO 3 STEPS but GO 3 STEPS, it will only match the red edges of the third step and the vertices at both ends.\nIn the basketballplayer dataset, the example is as follows:\n GO 3 STEPS FROM \"player100\" \\\n OVER * \\\n YIELD properties($$).name AS NAME, properties($$).age AS Age \\\n SAMPLE [1,2,3];\n GO 1 TO 3 STEPS FROM \"player100\" \\\n OVER * \\\n YIELD properties($$).name AS NAME, properties($$).age AS Age \\\n SAMPLE [2,2,2];","url":"8.clauses-and-options/sample/","type":"doc"},{"title":"ORDER BY","content":"The ORDER BY clause specifies the order of the rows in the output.\nNative nGQL: You must use a pipe (|) and an ORDER BY clause after YIELD clause.\nOpenCypher style: No pipes are permitted. The ORDER BY clause follows a RETURN clause.\nThere are two order options:\nASC: Ascending. ASC is the default order.\nDESC: Descending.\nNative nGQL Syntax\n\n| ORDER BY [ASC | DESC] [, [ASC | DESC] ...];\nExamples\n FETCH PROP ON player \"player100\", \"player101\", \"player102\", \"player103\" \\\n YIELD player.age AS age, player.name AS name \\\n | ORDER BY $-.age ASC, $-.name DESC;\n $var = GO FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS dst; \\\n ORDER BY $var.dst DESC;\nOpenCypher Syntax\n\nORDER BY [ASC | DESC] [, [ASC | DESC] ...];\nExamples\n MATCH (v:player) RETURN v.player.name AS Name, v.player.age AS Age \\\n ORDER BY Name DESC;\n...\n# In the following example, nGQL sorts the rows by age first. If multiple people are of the same age, nGQL will then sort them by name.\n MATCH (v:player) RETURN v.player.age AS Age, v.player.name AS Name \\\n ORDER BY Age DESC, Name ASC;\n...\nOrder of NULL values\nnGQL lists NULL values at the end of the output for ascending sorting, and at the start for descending sorting.\n MATCH (v:player{name:\"Tim Duncan\"}) --> (v2) \\\n RETURN v2.player.name AS Name, v2.player.age AS Age \\\n ORDER BY Age;\n MATCH (v:player{name:\"Tim Duncan\"}) --> (v2) \\\n RETURN v2.player.name AS Name, v2.player.age AS Age \\\n ORDER BY Age DESC;","url":"8.clauses-and-options/order-by/","type":"doc"},{"title":"RETURN","content":"The RETURN clause defines the output of an nGQL query. To return multiple fields, separate them with commas.\nRETURN can lead a clause or a statement:\nA RETURN clause can work in openCypher statements in nGQL, such as MATCH or UNWIND.\nA RETURN statement can work independently to output the result of an expression.\nOpenCypher compatibility\nThis topic applies to the openCypher syntax in nGQL only. For native nGQL, use YIELD.\nRETURN does not support the following openCypher features yet.\nReturn variables with uncommon characters, for example:\nMATCH (`non-english_characters`:player) \\\nRETURN `non-english_characters`;\nSet a pattern in the RETURN clause and return all elements that this pattern matches, for example:\nMATCH (v:player) \\\nRETURN (v)-[e]->(v2);\nMap order description\nWhen RETURN returns the map data structure, the order of key-value pairs is undefined.\n RETURN {age: 32, name: \"Marco Belinelli\"};\n RETURN {zage: 32, name: \"Marco Belinelli\"};\nReturn vertices or edges\nUse the RETURN { | } to return vertices and edges all information.\n// Return vertices\n MATCH (v:player) \\\n RETURN v;\n...\n// Return edges\n MATCH (v:player)-[e]->() \\\n RETURN e;\n...\nReturn VIDs\nUse the id() function to retrieve VIDs.\n MATCH (v:player{name:\"Tim Duncan\"}) \\\n RETURN id(v);\nReturn Tag\nUse the labels() function to return the list of tags on a vertex.\n MATCH (v:player{name:\"Tim Duncan\"}) \\\n RETURN labels(v);\nTo retrieve the nth element in the labels(v) list, use labels(v)[n-1]. The following example shows how to use labels(v)[0] to return the first tag in the list.\n MATCH (v:player{name:\"Tim Duncan\"}) \\\n RETURN labels(v)[0];\nReturn properties\nWhen returning properties of a vertex, it is necessary to specify the tag to which the properties belong because a vertex can have multiple tags and the same property name can appear on different tags.\nIt is possible to specify the tag of a vertex to return all properties of that tag, or to specify both the tag and a property name to return only that property of the tag.\n MATCH (v:player) \\\n RETURN v.player, v.player.name, v.player.age \\\n LIMIT 3;\nWhen returning edge properties, it is not necessary to specify the edge type to which the properties belong, because an edge can only have one edge type.\n// Return the property of a vertex\n MATCH p=(v:player{name:\"Tim Duncan\"})-[]->(v2) \\\n RETURN properties(v2);\n// Return the property of an edge\n MATCH (v:player{name:\"Tim Duncan\"})-[e]->() \\\n RETURN e.start_year, e.degree \\\nReturn edge type\nUse the type() function to return the matched edge types.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[e]->() \\\n RETURN DISTINCT type(e);\nReturn paths\nUse RETURN to return all the information of the matched paths.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[*3]->() \\\n RETURN p;\n...\nReturn vertices in a path\nUse the nodes() function to return all vertices in a path.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[]->(v2) \\\n RETURN nodes(p);\nReturn edges in a path\nUse the relationships() function to return all edges in a path.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[]->(v2) \\\n RETURN relationships(p);\nReturn path length\nUse the length() function to return the length of a path.\n MATCH p=(v:player{name:\"Tim Duncan\"})-[*..2]->(v2) \\\n RETURN p AS Paths, length(p) AS Length;\nReturn all elements\nTo return all the elements that this pattern matches, use an asterisk (*).\n MATCH (v:player{name:\"Tim Duncan\"}) \\\n RETURN *;\n MATCH (v:player{name:\"Tim Duncan\"})-[e]->(v2) \\\n RETURN *;\nRename a field\nUse the AS syntax to rename a field in the output.\n MATCH (v:player{name:\"Tim Duncan\"})-[:serve]->(v2) \\\n RETURN v2.team.name AS Team;\n RETURN \"Amber\" AS Name;\nReturn a non-existing property\nIf a property matched does not exist, NULL is returned.\n MATCH (v:player{name:\"Tim Duncan\"})-[e]->(v2) \\\n RETURN v2.player.name, type(e), v2.player.age;\nReturn expression results\nTo return the results of expressions such as literals, functions, or predicates, set them in a RETURN clause.\n MATCH (v:player{name:\"Tony Parker\"})-->(v2:player) \\\n RETURN DISTINCT v2.player.name, \"Hello\"+\" graphs!\", v2.player.age > 35;\n RETURN 1+1;\n RETURN 1- -1;\n RETURN 3 > 1;\n RETURN 1+1, rand32(1, 5);\nReturn unique fields\nUse DISTINCT to remove duplicate fields in the result set.\n# Before using DISTINCT.\n MATCH (v:player{name:\"Tony Parker\"})--(v2:player) \\\n RETURN v2.player.name, v2.player.age;\n# After using DISTINCT.\n MATCH (v:player{name:\"Tony Parker\"})--(v2:player) \\\n RETURN DISTINCT v2.player.name, v2.player.age;","url":"8.clauses-and-options/return/","type":"doc"},{"title":"TTL","content":"TTL (Time To Live) is a mechanism in NebulaGraph that defines the lifespan of data. Once the data reaches its predefined lifespan, it is automatically deleted from the database. This feature is particularly suitable for data that only needs temporary storage, such as temporary sessions or cached data.\nOpenCypher Compatibility\nThis topic applies to native nGQL only.\nPrecautions\nYou CANNOT modify a property schema with TTL options on it.\nTTL options and indexes have coexistence issues.\nTTL options and indexes CANNOT coexist on a tag or an edge type. If there is an index on a property, you cannot set TTL options on other properties.\nIf there are TTL options on a tag, an edge type, or a property, you can still add an index on them.\nTTL options\nThe native nGQL TTL feature has the following options.\nOption\nDescription\nttl_col\nSpecifies an existing property to set a lifespan on. The data type of the property must be int or timestamp.\nttl_duration\nSpecifies the timeout adds-on value in seconds. The value must be a non-negative int64 number. A property expires if the sum of its value and the ttl_duration value is smaller than the current timestamp. If the ttl_duration value is 0, the property never expires.You can set ttl_use_ms to true in the configuration file nebula-storaged.conf (default path: /usr/local/nightly/etc/) to set the default unit to milliseconds.\nUse TTL options\nYou must use the TTL options together to set a lifespan on a property.\nBefore using the TTL feature, you must first create a timestamp or integer property and specify it in the TTL options. NebulaGraph will not automatically create or manage this timestamp property for you.\nWhen inserting the value of the timestamp or integer property, it is recommended to use the now() function or the current timestamp to represent the present time.\nSet a timeout if a tag or an edge type exists\nIf a tag or an edge type is already created, to set a timeout on a property bound to the tag or edge type, use ALTER to update the tag or edge type.\n# Create a tag.\n CREATE TAG IF NOT EXISTS t1 (a timestamp);\n# Use ALTER to update the tag and set the TTL options.\n ALTER TAG t1 TTL_COL = \"a\", TTL_DURATION = 5;\n# Insert a vertex with tag t1. The vertex expires 5 seconds after the insertion.\n INSERT VERTEX t1(a) VALUES \"101\":(now());\nSet a timeout when creating a tag or an edge type\nUse TTL options in the CREATE statement to set a timeout when creating a tag or an edge type. For more information, see CREATE TAG and CREATE EDGE.\n# Create a tag and set the TTL options.\n CREATE TAG IF NOT EXISTS t2(a int, b int, c string) TTL_DURATION= 100, TTL_COL = \"a\";\n# Insert a vertex with tag t2. The timeout timestamp is 1648197238 (1648197138 + 100).\n INSERT VERTEX t2(a, b, c) VALUES \"102\":(1648197138, 30, \"Hello\");\nData expiration and deletion\nVertex property expiration\nVertex property expiration has the following impact.\nIf a vertex has only one tag, once a property of the vertex expires, the vertex expires.\nIf a vertex has multiple tags, once a property of the vertex expires, properties bound to the same tag with the expired property also expire, but the vertex does not expire and other properties of it remain untouched.\nEdge property expiration\nSince an edge can have only one edge type, once an edge property expires, the edge expires.\nData deletion\nThe expired data are still stored on the disk, but queries will filter them out.\nNebulaGraph automatically deletes the expired data and reclaims the disk space during the next compaction.\nRemove a timeout\nTo disable TTL and remove the timeout on a property, you can use the following approaches.\nDrop the property with the timeout. ALTER TAG t1 DROP (a);\nSet ttl_col to an empty string. ALTER TAG t1 TTL_COL = \"\";\nSet ttl_duration to 0. This operation keeps the TTL options and prevents the property from expiring and the property schema from being modified. ALTER TAG t1 TTL_DURATION = 0;","url":"8.clauses-and-options/ttl-options/","type":"doc"},{"title":"WHERE","content":"The WHERE clause filters the output by conditions.\nThe WHERE clause usually works in the following queries:\nNative nGQL: such as GO and LOOKUP.\nOpenCypher syntax: such as MATCH and WITH.\nOpenCypher compatibility\nFiltering on edge rank is a native nGQL feature. To retrieve the rank value in openCypher statements, use the rank() function, such as MATCH (:player)-[e:follow]->() RETURN rank(e);.\nBasic usage\nDefine conditions with boolean operators\nUse the boolean operators NOT, AND, OR, and XOR to define conditions in WHERE clauses. For the precedence of the operators, see Precedence.\n MATCH (v:player) \\\n WHERE v.player.name == \"Tim Duncan\" \\\n XOR (v.player.age < 30 AND v.player.name == \"Yao Ming\") \\\n OR NOT (v.player.name == \"Yao Ming\" OR v.player.name == \"Tim Duncan\") \\\n RETURN v.player.name, v.player.age;\n...\n GO FROM \"player100\" \\\n OVER follow \\\n WHERE properties(edge).degree > 90 \\\n OR properties($$).age != 33 \\\n AND properties($$).name != \"Tony Parker\" \\\n YIELD properties($$);\nFilter on properties\nUse vertex or edge properties to define conditions in WHERE clauses.\nFilter on a vertex property: MATCH (v:player)-[e]->(v2) \\\n WHERE v2.player.age < 25 \\\n RETURN v2.player.name, v2.player.age;\n GO FROM \"player100\" OVER follow \\\n WHERE $^.player.age >= 42 \\\n YIELD dst(edge);\nFilter on an edge property: MATCH (v:player)-[e]->() \\\n WHERE e.start_year < 2000 \\\n RETURN DISTINCT v.player.name, v.player.age;\n...\n GO FROM \"player100\" OVER follow \\\n WHERE follow.degree > 90 \\\n YIELD dst(edge);\nFilter on dynamically-calculated properties\n MATCH (v:player) \\\n WHERE v[toLower(\"AGE\")] < 21 \\\n RETURN v.player.name, v.player.age;\nFilter on existing properties\n MATCH (v:player) \\\n WHERE exists(v.player.age) \\\n RETURN v.player.name, v.player.age;\n...\nFilter on edge rank\nIn nGQL, if a group of edges has the same source vertex, destination vertex, and properties, the only thing that distinguishes them is the rank. Use rank conditions in WHERE clauses to filter such edges.\n# The following example creates test data.\n CREATE SPACE IF NOT EXISTS test (vid_type=FIXED_STRING(30));\n USE test;\n CREATE EDGE IF NOT EXISTS e1(p1 int);\n CREATE TAG IF NOT EXISTS person(p1 int);\n INSERT VERTEX person(p1) VALUES \"1\":(1);\n INSERT VERTEX person(p1) VALUES \"2\":(2);\n INSERT EDGE e1(p1) VALUES \"1\"->\"2\"@0:(10);\n INSERT EDGE e1(p1) VALUES \"1\"->\"2\"@1:(11);\n INSERT EDGE e1(p1) VALUES \"1\"->\"2\"@2:(12);\n INSERT EDGE e1(p1) VALUES \"1\"->\"2\"@3:(13);\n INSERT EDGE e1(p1) VALUES \"1\"->\"2\"@4:(14);\n INSERT EDGE e1(p1) VALUES \"1\"->\"2\"@5:(15);\n INSERT EDGE e1(p1) VALUES \"1\"->\"2\"@6:(16);\n# The following example use rank to filter edges and retrieves edges with a rank greater than 2.\n GO FROM \"1\" \\\n OVER e1 \\\n WHERE rank(edge) > 2 \\\n YIELD src(edge), dst(edge), rank(edge) AS Rank, properties(edge).p1 | \\\n ORDER BY $-.Rank DESC;\n# Filter edges by rank. Find follow edges with rank equal to 0.\n MATCH (v)-[e:follow]->() \\\n WHERE rank(e)==0 \\\n RETURN *;\n...\nFilter on pattern\n MATCH (v:player{name:\"Tim Duncan\"})-[e]->(t) \\\n WHERE (v)-[e]->(t:team) \\\n RETURN (v)-->();\n MATCH (v:player{name:\"Tim Duncan\"})-[e]->(t) \\\n WHERE NOT (v)-[e]->(t:team) \\\n RETURN (v)-->();\nFilter on strings\nUse STARTS WITH, ENDS WITH, or CONTAINS in WHERE clauses to match a specific part of a string. String matching is case-sensitive.\nSTARTS WITH\nSTARTS WITH will match the beginning of a string.\nThe following example uses STARTS WITH \"T\" to retrieve the information of players whose name starts with T.\n MATCH (v:player) \\\n WHERE v.player.name STARTS WITH \"T\" \\\n RETURN v.player.name, v.player.age;\nIf you use STARTS WITH \"t\" in the preceding statement, an empty set is returned because no name in the dataset starts with the lowercase t.\n MATCH (v:player) \\\n WHERE v.player.name STARTS WITH \"t\" \\\n RETURN v.player.name, v.player.age;\nEmpty set (time spent 5080/6474 us)\nENDS WITH\nENDS WITH will match the ending of a string.\nThe following example uses ENDS WITH \"r\" to retrieve the information of players whose name ends with r.\n MATCH (v:player) \\\n WHERE v.player.name ENDS WITH \"r\" \\\n RETURN v.player.name, v.player.age;\nCONTAINS\nCONTAINS will match a certain part of a string.\nThe following example uses CONTAINS \"Pa\" to match the information of players whose name contains Pa.\n MATCH (v:player) \\\n WHERE v.player.name CONTAINS \"Pa\" \\\n RETURN v.player.name, v.player.age;\nNegative string matching\nYou can use the boolean operator NOT to negate a string matching condition.\n MATCH (v:player) \\\n WHERE NOT v.player.name ENDS WITH \"R\" \\\n RETURN v.player.name, v.player.age;\n...\nFilter on lists\nMatch values in a list\nUse the IN operator to check if a value is in a specific list.\n MATCH (v:player) \\\n WHERE v.player.age IN range(20,25) \\\n RETURN v.player.name, v.player.age;\n LOOKUP ON player \\\n WHERE player.age IN [25,28] \\\n YIELD properties(vertex).name, properties(vertex).age;\nMatch values not in a list\nUse NOT before IN to rule out the values in a list.\n MATCH (v:player) \\\n WHERE v.player.age NOT IN range(20,25) \\\n RETURN v.player.name AS Name, v.player.age AS Age \\\n ORDER BY Age;\n...","url":"8.clauses-and-options/where/","type":"doc"},{"title":"YIELD","content":"YIELD defines the output of an nGQL query.\nYIELD can lead a clause or a statement:\nA YIELD clause works in nGQL statements such as GO, FETCH, or LOOKUP and must be defined to return the result.\nA YIELD statement works in a composite query or independently.\nOpenCypher compatibility\nThis topic applies to native nGQL only. For the openCypher syntax, use RETURN.\nYIELD has different functions in openCypher and nGQL.\nIn openCypher, YIELD is used in the CALL[…YIELD] clause to specify the output of the procedure call.\nIn nGQL, YIELD works like RETURN in openCypher.\nYIELD clauses\nSyntax\nYIELD [DISTINCT] [AS ] [, [AS ] ...];\nParameter\nDescription\nDISTINCT\nAggregates the output and makes the statement return a distinct result set.\ncol\nA field to be returned. If no alias is set, col will be a column name in the output.\nalias\nAn alias for col. It is set after the keyword AS and will be a column name in the output.\nUse a YIELD clause in a statement\nUse YIELD with GO: GO FROM \"player100\" OVER follow \\\n YIELD properties($$).name AS Friend, properties($$).age AS Age;\nUse YIELD with FETCH: FETCH PROP ON player \"player100\" \\\n YIELD properties(vertex).name;\nUse YIELD with LOOKUP: LOOKUP ON player WHERE player.name == \"Tony Parker\" \\\n YIELD properties(vertex).name, properties(vertex).age;\nYIELD statements\nSyntax\nYIELD [DISTINCT] [AS ] [, [AS ] ...]\n[WHERE ];\nParameter\nDescription\nDISTINCT\nAggregates the output and makes the statement return a distinct result set.\ncol\nA field to be returned. If no alias is set, col will be a column name in the output.\nalias\nAn alias for col. It is set after the keyword AS and will be a column name in the output.\nconditions\nConditions set in a WHERE clause to filter the output. For more information, see WHERE.\nUse a YIELD statement in a composite query\nIn a composite query, a YIELD statement accepts, filters, and modifies the result set of the preceding statement, and then outputs it.\nThe following query finds the players that \"player100\" follows and calculates their average age.\n GO FROM \"player100\" OVER follow \\\n YIELD dst(edge) AS ID \\\n | FETCH PROP ON player $-.ID \\\n YIELD properties(vertex).age AS Age \\\n | YIELD AVG($-.Age) as Avg_age, count(*)as Num_friends;\nThe following query finds the players that \"player101\" follows with the follow degrees greater than 90.\n $var1 = GO FROM \"player101\" OVER follow \\\n YIELD properties(edge).degree AS Degree, dst(edge) as ID; \\\n YIELD $var1.ID AS ID WHERE $var1.Degree > 90;\nThe following query finds the vertices in the player that are older than 30 and younger than 32, and returns the de-duplicate results.\n LOOKUP ON player \\\n WHERE player.age < 32 and player.age >30 \\\n YIELD DISTINCT properties(vertex).age as v;\nUse a standalone YIELD statement\nA YIELD statement can calculate a valid expression and output the result.\n YIELD rand32(1, 6);\n YIELD \"Hel\" + \"\\tlo\" AS string1, \", World!\" AS string2;\n YIELD hash(\"Tim\") % 100;\n YIELD \\\n CASE 2+3 \\\n WHEN 4 THEN 0 \\\n WHEN 5 THEN 1 \\\n ELSE -1 \\\n END \\\n AS result;\n YIELD 1- -1;","url":"8.clauses-and-options/yield/","type":"doc"},{"title":"WITH","content":"The WITH clause can retrieve the output from a query part, process it, and pass it to the next query part as the input.\nOpenCypher compatibility\nThis topic applies to openCypher syntax only.\nCombine statements and form a composite query\nUse a WITH clause to combine statements and transfer the output of a statement as the input of another statement.\nExample 1\nThe following statement:\nMatches a path.\nOutputs all the vertices on the path to a list with the nodes() function.\nUnwinds the list into rows.\nRemoves duplicated vertices and returns a set of distinct vertices.\n MATCH p=(v:player{name:\"Tim Duncan\"})--() \\\n WITH nodes(p) AS n \\\n UNWIND n AS n1 \\\n RETURN DISTINCT n1;\nExample 2\nThe following statement:\nMatches the vertex with the VID player100.\nOutputs all the tags of the vertex into a list with the labels() function.\nUnwinds the list into rows.\nReturns the output.\n MATCH (v) \\\n WHERE id(v)==\"player100\" \\\n WITH labels(v) AS tags_unf \\\n UNWIND tags_unf AS tags_f \\\n RETURN tags_f;\nFilter composite queries\nWITH can work as a filter in the middle of a composite query.\n MATCH (v:player)-->(v2:player) \\\n WITH DISTINCT v2 AS v2, v2.player.age AS Age \\\n ORDER BY Age \\\n WHERE Age<25 \\\n RETURN v2.player.name AS Name, Age;\nProcess the output before using collect()\nUse a WITH clause to sort and limit the output before using collect() to transform the output into a list.\n MATCH (v:player) \\\n WITH v.player.name AS Name \\\n ORDER BY Name DESC \\\n LIMIT 3 \\\n RETURN collect(Name);\nUse with RETURN\nSet an alias using a WITH clause, and then output the result through a RETURN clause.\n WITH [1, 2, 3] AS `list` RETURN 3 IN `list` AS r;\n WITH 4 AS one, 3 AS two RETURN one > two AS result;","url":"8.clauses-and-options/with/","type":"doc"},{"title":"UNWIND","content":"UNWIND transform a list into a sequence of rows.\nUNWIND can be used as an individual statement or as a clause within a statement.\nUNWIND statement\nSyntax\nUNWIND AS ;\nExamples\nTo transform a list.\n UNWIND [1,2,3] AS n RETURN n;\nUNWIND clause\nSyntax\nThe UNWIND clause in native nGQL statements.\n | UNWIND $-. AS <|> ;\nThe UNWIND clause in openCypher statements.\n UNWIND AS ；\nExamples\nTo transform a list of duplicates into a unique set of rows using WITH DISTINCT in a UNWIND clause.\n// Transform the list `[1,1,2,2,3,3]` into a unique set of rows, sort the rows, and then transform the rows into a list of unique values.\n WITH [1,1,2,2,3,3] AS n \\\n UNWIND n AS r \\\n WITH DISTINCT r AS r \\\n ORDER BY r \\\n RETURN collect(r);\nTo use an UNWIND clause in a MATCH statement.\n// Get a list of the vertices in the matched path, transform the list into a unique set of rows, and then transform the rows into a list. \n MATCH p=(v:player{name:\"Tim Duncan\"})--(v2) \\\n WITH nodes(p) AS n \\\n UNWIND n AS r \\\n WITH DISTINCT r AS r \\\n RETURN collect(r);\n|(\"player109\" :player{age: 34, name: \"Tiago Splitter\"}), (\"player108\" :player{age: 36, name: \"Boris Diaw\"})] | \nTo use an UNWIND clause in a GO statement.\n// Query the vertices in a list for the corresponding edges with a specified statement.\n YIELD ['player101', 'player100'] AS a | UNWIND $-.a AS b | GO FROM $-.b OVER follow YIELD edge AS e;\nTo use an UNWIND clause in a LOOKUP statement.\n// Find all the properties of players whose age is greater than 46, get a list of unique properties, and then transform the list into rows. \n LOOKUP ON player \\\n WHERE player.age > 46 \\\n YIELD DISTINCT keys(vertex) as p | UNWIND $-.p as a | YIELD $-.a AS a;\nTo use an UNWIND clause in a FETCH statement. \n// Query player101 for all tags related to player101, get a list of the tags and then transform the list into rows.\n CREATE TAG hero(like string, height int);\n INSERT VERTEX hero(like, height) VALUES \"player101\":(\"deep\", 182);\n FETCH PROP ON * \"player101\" \\\n YIELD tags(vertex) as t | UNWIND $-.t as a | YIELD $-.a AS a;\nTo use an UNWIND clause in a GET SUBGRAPH statement. \n// Get the subgraph including outgoing and incoming serve edges within 0~2 hops from/to player100, and transform the result into rows.\n GET SUBGRAPH 2 STEPS FROM \"player100\" BOTH serve \\\n YIELD edges as e | UNWIND $-.e as a | YIELD $-.a AS a;\nTo use an UNWIND clause in a FIND PATH statement.\n// Find all the vertices in the shortest path from player101 to team204 along the serve edge, and transform the result into rows. \n FIND SHORTEST PATH FROM \"player101\" TO \"team204\" OVER serve \\\n YIELD path as p | YIELD nodes($-.p) AS nodes | UNWIND $-.nodes AS a | YIELD $-.a AS a;","url":"8.clauses-and-options/unwind/","type":"doc"},{"title":"CREATE SPACE","content":"Graph spaces are used to store data in a physically isolated way in NebulaGraph, which is similar to the database concept in MySQL. The CREATE SPACE statement can create a new graph space or clone the schema of an existing graph space.\nPrerequisites\nOnly the God role can use the CREATE SPACE statement. For more information, see AUTHENTICATION.\nSyntax\nCreate graph spaces\nCREATE SPACE [IF NOT EXISTS] (\n [partition_num = ,]\n [replica_factor = ,]\n vid_type = {FIXED_STRING() | INT[64]}\n [COMMENT = '']\nParameter\nDescription\nIF NOT EXISTS\nDetects if the related graph space exists. If it does not exist, a new one will be created. The graph space existence detection here only compares the graph space name (excluding properties).\n\n1. Uniquely identifies a graph space in a NebulaGraph instance. 2. Space names cannot be modified after they are set. 3. By default, the name only supports 1-4 byte UTF-8 encoded characters, including English letters (case sensitive), numbers, Chinese characters, etc. However, it cannot include special characters other than the underscore (_), and cannot start with a number. 4. To use special characters, reserved keywords, or start with a number, quote the entire name with backticks (`) and do not include periods (.) within the pair of backticks (`). For more information, see Keywords and reserved words. Note: If you name a space in Chinese and encounter a SyntaxError, you need to quote the Chinese characters with backticks (`).\npartition_num\nSpecifies the number of partitions in each replica. The suggested value is 20 times (2 times for HDD) the number of the hard disks in the cluster. For example, if you have three hard disks in the cluster, we recommend that you set 60 partitions. The default value is 100.\nreplica_factor\nSpecifies the number of replicas in the cluster. The suggested number is 3 in a production environment and 1 in a test environment. The replica number must be an odd number for the need of quorum-based voting. The default value is 1.\nvid_type\nA required parameter. Specifies the VID type in a graph space. Available values are FIXED_STRING(N) and INT64. INT equals to INT64. `FIXED_STRING() specifies the VID as a string, while INT64 specifies it as an integer. N represents the maximum length of the VIDs. If you set a VID that is longer than N bytes, NebulaGraph throws an error. Note, for UTF-8 chars, the length may vary in different cases, i.e. a UTF-8 Chinese char is 3 byte, this means 11 Chinese chars(length-33) will exeed a FIXED_STRING(32) vid defination.\nCOMMENT\nThe remarks of the graph space. The maximum length is 256 bytes. By default, there is no comments on a space.\nClone graph spaces\nCREATE SPACE [IF NOT EXISTS] AS ;\nParameter\nDescription\nIF NOT EXISTS\nDetects if the new graph space exists. If it does not exist, the new one will be created. The graph space existence detection here only compares the graph space name (excluding properties).\n\nThe name of the graph space that is newly created. By default, the space name only supports 1-4 byte UTF-8 encoded characters, including English letters (case sensitive), numbers, Chinese characters, etc. But special characters can only use underscore, and cannot start with a number. To use special characters, reserved keywords, or start with a number, quote the entire name with backticks (`) and cannot use periods (.). For more information, see Keywords and reserved words. When a new graph space is created, the schema of the old graph space will be cloned, including its parameters (the number of partitions and replicas, etc.), Tag, Edge type and native indexes. Note: If you name a space in Chinese and encounter a SyntaxError, you need to quote the Chinese characters with backticks (`).\n\nThe name of the graph space that already exists.\nExamples\n# The following example creates a graph space with a specified VID type and the maximum length. Other fields still use the default values.\n CREATE SPACE IF NOT EXISTS my_space_1 (vid_type=FIXED_STRING(30));\n# The following example creates a graph space with a specified partition number, replica number, and VID type.\n CREATE SPACE IF NOT EXISTS my_space_2 (partition_num=15, replica_factor=1, vid_type=FIXED_STRING(30));\n# The following example creates a graph space with a specified partition number, replica number, and VID type, and adds a comment on it.\n CREATE SPACE IF NOT EXISTS my_space_3 (partition_num=15, replica_factor=1, vid_type=FIXED_STRING(30)) comment=\"Test the graph space\";\n# Clone a graph space.\n CREATE SPACE IF NOT EXISTS my_space_4 as my_space_3;\n SHOW CREATE SPACE my_space_4;\nImplementation of the operation\nCheck partition distribution\nOn some large clusters, the partition distribution is possibly unbalanced because of the different startup times. You can run the following command to do a check of the machine distribution.\n SHOW HOSTS;\nTo balance the request loads, use the following command.\n BALANCE LEADER;\n SHOW HOSTS;","url":"9.space-statements/1.create-space/","type":"doc"},{"title":"USE","content":"USE specifies a graph space as the current working graph space for subsequent queries.\nPrerequisites\nRunning the USE statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nSyntax\nUSE ;\nExamples\n# The following example creates two sample spaces.\n CREATE SPACE IF NOT EXISTS space1 (vid_type=FIXED_STRING(30));\n CREATE SPACE IF NOT EXISTS space2 (vid_type=FIXED_STRING(30));\n# The following example specifies space1 as the current working graph space.\n USE space1;\n# The following example specifies space2 as the current working graph space. Hereafter, you cannot read any data from space1, because these vertices and edges being traversed have no relevance with space1.\n USE space2;","url":"9.space-statements/2.use-space/","type":"doc"},{"title":"SHOW SPACES","content":"SHOW SPACES lists all the graph spaces in the NebulaGraph examples.\nSyntax\nSHOW SPACES;\nExample\n SHOW SPACES;\nTo create graph spaces, see CREATE SPACE.","url":"9.space-statements/3.show-spaces/","type":"doc"},{"title":"DESCRIBE SPACE","content":"DESCRIBE SPACE returns the information about the specified graph space.\nSyntax\nYou can use DESC instead of DESCRIBE for short.\nDESC[RIBE] SPACE ;\nThe DESCRIBE SPACE statement is different from the SHOW SPACES statement. For details about SHOW SPACES, see SHOW SPACES.\nExample\n DESCRIBE SPACE basketballplayer;","url":"9.space-statements/4.describe-space/","type":"doc"},{"title":"CLEAR SPACE","content":"CLEAR SPACE deletes the vertices and edges in a graph space, but does not delete the graph space itself and the schema information.\nPermission requirements\nOnly the God role has the permission to run CLEAR SPACE.\nCaution\nOnce cleared, the data CANNOT be recovered. Use CLEAR SPACE with caution.\nCLEAR SPACE is not an atomic operation. If an error occurs, re-run CLEAR SPACE to avoid data remaining.\nThe larger the amount of data in the graph space, the longer it takes to clear it. If the execution fails due to client connection timeout, increase the value of the storage_client_timeout_ms parameter in the Graph Service configuration.\nDuring the execution of CLEAR SPACE, writing data into the graph space is not automatically prohibited. Such write operations can result in incomplete data clearing, and the residual data can be damaged.\nSyntax\nCLEAR SPACE [IF EXISTS] ;\nParameter/Option\nDescription\nIF EXISTS\nCheck whether the graph space to be cleared exists. If it exists, continue to clear it. If it does not exist, the execution finishes, and a message indicating that the execution succeeded is displayed. If IF EXISTS is not set and the graph space does not exist, the CLEAR SPACE statement fails to execute, and an error occurs.\nspace_name\nThe name of the space to be cleared.\nExample:\nCLEAR SPACE basketballplayer;\nData reserved\nCLEAR SPACE does not delete the following data in a graph space:\nTag information.\nEdge type information.\nThe metadata of native indexes and full-text indexes.\nThe following example shows what CLEAR SPACE deletes and reserves.\n# Enter the graph space basketballplayer.\nnebula [(none)]> use basketballplayer;\nExecution succeeded\n# List tags and Edge types.\nnebula[basketballplayer]> SHOW TAGS;\nGot 2 rows\nnebula[basketballplayer]> SHOW EDGES;\nGot 2 rows\n# Submit a job to make statistics of the graph space.\nnebula[basketballplayer]> SUBMIT JOB STATS;\nGot 1 rows\n# Check the statistics.\nnebula[basketballplayer]> SHOW STATS;\nGot 6 rows\n# List tag indexes.\nnebula[basketballplayer]> SHOW TAG INDEXES;\nGot 2 rows\n# ----------------------- Dividing line for CLEAR SPACE -----------------------\n# Run CLEAR SPACE to clear the graph space basketballplayer.\nnebula[basketballplayer]> CLEAR SPACE basketballplayer;\nExecution succeeded\n# Update the statistics.\nnebula[basketballplayer]> SUBMIT JOB STATS;\nGot 1 rows\n# Check the statistics. The tags and edge types still exist, but all the vertices and edges are gone.\nnebula[basketballplayer]> SHOW STATS;\nGot 6 rows\n# Try to list the tag indexes. They still exist.\nnebula[basketballplayer]> SHOW TAG INDEXES;\nGot 2 rows (time spent 523/978 us)","url":"9.space-statements/6.clear-space/","type":"doc"},{"title":"DROP SPACE","content":"DROP SPACE deletes the specified graph space and everything in it.\nPrerequisites\nOnly the God role can use the DROP SPACE statement. For more information, see AUTHENTICATION.\nSyntax\nDROP SPACE [IF EXISTS] ;\nYou can use the IF EXISTS keywords when dropping spaces. These keywords automatically detect if the related graph space exists. If it exists, it will be deleted. Otherwise, no graph space will be deleted.\nFAQ\nQ: Why is my disk space not freed after executing the 'DROP SPACE' statement and deleting a graph space?\nA: For NebulaGraph version earlier than 3.1.0, DROP SPACE can only delete the specified logic graph space and does not delete the files and directories on the disk. To delete the files and directories on the disk, manually delete the corresponding file path. The file path is located in /data/storage/nebula/. The can be viewed via DESCRIBE SPACE {space_name}.","url":"9.space-statements/5.drop-space/","type":"doc"},{"title":"CREATE TAG","content":"CREATE TAG creates a tag with the given name in a graph space.\nOpenCypher compatibility\nTags in nGQL are similar to labels in openCypher. But they are also quite different. For example, the ways to create them are different.\nIn openCypher, labels are created together with vertices in CREATE statements.\nIn nGQL, tags are created separately using CREATE TAG statements. Tags in nGQL are more like tables in MySQL.\nPrerequisites\nRunning the CREATE TAG statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nSyntax\nTo create a tag in a specific graph space, you must specify the current working space with the USE statement.\nCREATE TAG [IF NOT EXISTS] \n [NULL | NOT NULL] [DEFAULT ] [COMMENT '']\n [{, [NULL | NOT NULL] [DEFAULT ] [COMMENT '']} ...] \n [TTL_DURATION = ]\n [TTL_COL = ]\n [COMMENT = ''];\nParameter\nDescription\nIF NOT EXISTS\nDetects if the tag that you want to create exists. If it does not exist, a new one will be created. The tag existence detection here only compares the tag names (excluding properties).\n\n1. Each tag name in the graph space must be unique. 2. Tag names cannot be modified after they are set. 3. By default, the name only supports 1-4 byte UTF-8 encoded characters, including English letters (case sensitive), numbers, Chinese characters, etc. However, it cannot include special characters other than the underscore (_), and cannot start with a number. 4. To use special characters, reserved keywords, or start with a number, quote the entire name with backticks (`) and do not include periods (.) within the pair of backticks (`). For more information, see Keywords and reserved words. Note: If you name a tag in Chinese and encounter a SyntaxError, you need to quote the Chinese characters with backticks (`).\n\nThe name of the property. It must be unique for each tag. The rules for permitted property names are the same as those for tag names.\n\nShows the data type of each property. For a full description of the property data types, see Data types and Boolean.\nNULL | NOT NULL\nSpecifies if the property supports NULL | NOT NULL. The default value is NULL.\nDEFAULT\nSpecifies a default value for a property. The default value can be a literal value or an expression supported by NebulaGraph. If no value is specified, the default value is used when inserting a new vertex.\nCOMMENT\nThe remarks of a certain property or the tag itself. The maximum length is 256 bytes. By default, there will be no comments on a tag.\nTTL_DURATION\nSpecifies the life cycle for the property. The property that exceeds the specified TTL expires. The expiration threshold is the TTL_COL value plus the TTL_DURATION. The default value of TTL_DURATION is 0. It means the data never expires.\nTTL_COL\nSpecifies the property to set a timeout on. The data type of the property must be int or timestamp. A tag can only specify one field as TTL_COL. For more information on TTL, see TTL options.\nExamples\n CREATE TAG IF NOT EXISTS player(name string, age int);\n# The following example creates a tag with no properties.\n CREATE TAG IF NOT EXISTS no_property(); \n# The following example creates a tag with a default value.\n CREATE TAG IF NOT EXISTS player_with_default(name string, age int DEFAULT 20);\n# In the following example, the TTL of the create_time field is set to be 100 seconds.\n CREATE TAG IF NOT EXISTS woman(name string, age int, \\\n married bool, salary double, create_time timestamp) \\\n TTL_DURATION = 100, TTL_COL = \"create_time\";\nImplementation of the operation\nTrying to use a newly created tag may fail because the creation of the tag is implemented asynchronously. To make sure the follow-up operations work as expected, Wait for two heartbeat cycles, i.e., 20 seconds.\nTo change the heartbeat interval, modify the heartbeat_interval_secs parameter in the configuration files for all services.","url":"10.tag-statements/1.create-tag/","type":"doc"},{"title":"DROP TAG","content":"DROP TAG drops a tag with the given name in the current working graph space.\nA vertex can have one or more tags.\nIf a vertex has only one tag, the vertex CANNOT be accessed after you drop it. The vertex will be dropped in the next compaction. But its edges are available, this operation will result in dangling edges.\nIf a vertex has multiple tags, the vertex is still accessible after you drop one of them. But all the properties defined by this dropped tag CANNOT be accessed.\nThis operation only deletes the Schema data. All the files or directories in the disk will not be deleted directly until the next compaction.\nPrerequisites\nRunning the DROP TAG statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nBefore you drop a tag, make sure that the tag does not have any indexes. Otherwise, the conflict error ([ERROR (-1005)]: Conflict!) will be returned when you run the DROP TAG statement. To drop an index, see DROP INDEX.\nSyntax\nDROP TAG [IF EXISTS] ;\nIF NOT EXISTS: Detects if the tag that you want to drop exists. Only when it exists will it be dropped.\ntag_name: Specifies the tag name that you want to drop. You can drop only one tag in one statement.\nExample\n CREATE TAG IF NOT EXISTS test(p1 string, p2 int);\n DROP TAG test;","url":"10.tag-statements/2.drop-tag/","type":"doc"},{"title":"ALTER TAG","content":"ALTER TAG alters the structure of a tag with the given name in a graph space. You can add or drop properties, and change the data type of an existing property. You can also set a TTL (Time-To-Live) on a property, or change its TTL duration.\nNotes\nRunning the ALTER TAG statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nBefore you alter properties for a tag, make sure that the properties are not indexed. If the properties contain any indexes, the conflict error [ERROR (-1005)]: Conflict! will occur when you ALTER TAG. For more information on dropping an index, see DROP INDEX.\nThe property name must be unique in a tag. If you add a property with the same name as an existing property or a dropped property, the operation fails.\nSyntax\nALTER TAG \n [[, alter_definition] ...]\n [ttl_definition [, ttl_definition] ... ]\n [COMMENT ''];\nalter_definition:\n| ADD (prop_name data_type [NULL | NOT NULL] [DEFAULT ] [COMMENT ''])\n| DROP (prop_name)\n| CHANGE (prop_name data_type [NULL | NOT NULL] [DEFAULT ] [COMMENT ''])\nttl_definition:\n TTL_DURATION = ttl_duration, TTL_COL = prop_name\ntag_name: Specifies the tag name that you want to alter. You can alter only one tag in one statement. Before you alter a tag, make sure that the tag exists in the current working graph space. If the tag does not exist, an error will occur when you alter it.\nMultiple ADD, DROP, and CHANGE clauses are permitted in a single ALTER TAG statement, separated by commas.\nWhen a property value is set to NOT NULL using ADD or CHANGE, a default value must be specified for the property, that is, the value of DEFAULT must be specified.\nWhen using CHANGE to modify the data type of a property:\nOnly the length of a FIXED_STRING or an INT can be increased. The length of a STRING or an INT cannot be decreased.\nOnly the data type conversions from FIXED_STRING to STRING and from FLOAT to DOUBLE are allowed.\nExamples\n CREATE TAG IF NOT EXISTS t1 (p1 string, p2 int);\n ALTER TAG t1 ADD (p3 int32, fixed_string(10));\n ALTER TAG t1 TTL_DURATION = 2, TTL_COL = \"p2\";\n ALTER TAG t1 COMMENT = 'test1';\n ALTER TAG t1 ADD (p5 double NOT NULL DEFAULT 0.4 COMMENT 'p5') COMMENT='test2';\n// Change the data type of p3 in the TAG t1 from INT32 to INT64, and that of p4 from FIXED_STRING(10) to STRING.\n ALTER TAG t1 CHANGE (p3 int64, p4 string);\n[ERROR(-1005)]: Unsupported!\nImplementation of the operation\nTrying to use a newly altered tag may fail because the alteration of the tag is implemented asynchronously. To make sure the follow-up operations work as expected, Wait for two heartbeat cycles, i.e., 20 seconds.\nTo change the heartbeat interval, modify the heartbeat_interval_secs parameter in the configuration files for all services.","url":"10.tag-statements/3.alter-tag/","type":"doc"},{"title":"SHOW TAGS","content":"The SHOW TAGS statement shows the name of all tags in the current graph space.\nYou do not need any privileges for the graph space to run the SHOW TAGS statement. But the returned results are different based on role privileges.\nSyntax\nSHOW TAGS;\nExamples\n SHOW TAGS;","url":"10.tag-statements/4.show-tags/","type":"doc"},{"title":"DESCRIBE TAG","content":"DESCRIBE TAG returns the information about a tag with the given name in a graph space, such as field names, data type, and so on.\nPrerequisite\nRunning the DESCRIBE TAG statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nSyntax\nDESC[RIBE] TAG ;\nYou can use DESC instead of DESCRIBE for short.\nExample\n DESCRIBE TAG player;","url":"10.tag-statements/5.describe-tag/","type":"doc"},{"title":"DELETE TAG","content":"DELETE TAG deletes a tag with the given name on a specified vertex.\nPrerequisites\nRunning the DELETE TAG statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nSyntax\nDELETE TAG FROM ;\ntag_name_list: The names of the tags you want to delete. Multiple tags are separated with commas (,). * means all tags.\nVID: The VIDs of the vertices from which you want to delete the tags. Multiple VIDs are separated with commas (,). \nExample\n CREATE TAG IF NOT EXISTS test1(p1 string, p2 int);\n CREATE TAG IF NOT EXISTS test2(p3 string, p4 int);\n INSERT VERTEX test1(p1, p2),test2(p3, p4) VALUES \"test\":(\"123\", 1, \"456\", 2);\n FETCH PROP ON * \"test\" YIELD vertex AS v;\n DELETE TAG test1 FROM \"test\";\n FETCH PROP ON * \"test\" YIELD vertex AS v;\n DELETE TAG * FROM \"test\";\n FETCH PROP ON * \"test\" YIELD vertex AS v;","url":"10.tag-statements/6.delete-tag/","type":"doc"},{"title":"Add and delete tags","content":"OpenCypher has the features of SET label and REMOVE label to speed up the process of querying or labeling.\nNebulaGraph achieves the same operations by creating and inserting tags to an existing vertex, which can quickly query vertices based on the tag name. Users can also run DELETE TAG to delete some vertices that are no longer needed.\nExamples\nFor example, in the basketballplayer data set, some basketball players are also team shareholders. Users can create an index for the shareholder tag shareholder for quick search. If the player is no longer a shareholder, users can delete the shareholder tag of the corresponding player by DELETE TAG.\n//This example creates the shareholder tag and index.\n CREATE TAG IF NOT EXISTS shareholder();\n CREATE TAG INDEX IF NOT EXISTS shareholder_tag on shareholder();\n//This example adds a tag on the vertex.\n INSERT VERTEX shareholder() VALUES \"player100\":();\n INSERT VERTEX shareholder() VALUES \"player101\":();\n//This example queries all the shareholders.\n MATCH (v:shareholder) RETURN v;\n LOOKUP ON shareholder YIELD id(vertex);\n//In this example, the \"player100\" is no longer a shareholder.\n DELETE TAG shareholder FROM \"player100\";\n LOOKUP ON shareholder YIELD id(vertex);","url":"10.tag-statements/improve-query-by-tag-index/","type":"doc"},{"title":"CREATE EDGE","content":"CREATE EDGE creates an edge type with the given name in a graph space.\nOpenCypher compatibility\nEdge types in nGQL are similar to relationship types in openCypher. But they are also quite different. For example, the ways to create them are different.\nIn openCypher, relationship types are created together with vertices in CREATE statements.\nIn nGQL, edge types are created separately using CREATE EDGE statements. Edge types in nGQL are more like tables in MySQL.\nPrerequisites\nRunning the CREATE EDGE statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nSyntax\nTo create an edge type in a specific graph space, you must specify the current working space with the USE statement.\nCREATE EDGE [IF NOT EXISTS] \n [NULL | NOT NULL] [DEFAULT ] [COMMENT '']\n [{, [NULL | NOT NULL] [DEFAULT ] [COMMENT '']} ...] \n [TTL_DURATION = ]\n [TTL_COL = ]\n [COMMENT = ''];\nParameter\nDescription\nIF NOT EXISTS\nDetects if the edge type that you want to create exists. If it does not exist, a new one will be created. The edge type existence detection here only compares the edge type names (excluding properties).\n\n1. The edge type name must be unique in a graph space. 2. Once the edge type name is set, it can not be altered. 3. By default, the name only supports 1-4 byte UTF-8 encoded characters, including English letters (case sensitive), numbers, Chinese characters, etc. However, it cannot include special characters other than the underscore (_), and cannot start with a number. 4. To use special characters, reserved keywords, or start with a number, quote the entire name with backticks (`) and do not include periods (.) within the pair of backticks (`). For more information, see Keywords and reserved words. Note: If you name an edge type in Chinese and encounter a SyntaxError, you need to quote the Chinese characters with backticks (`).\n\nThe name of the property. It must be unique for each edge type. The rules for permitted property names are the same as those for edge type names.\n\nShows the data type of each property. For a full description of the property data types, see Data types and Boolean.\nNULL | NOT NULL\nSpecifies if the property supports NULL | NOT NULL. The default value is NULL. DEFAULT must be specified if NOT NULL is set.\nDEFAULT\nSpecifies a default value for a property. The default value can be a literal value or an expression supported by NebulaGraph. If no value is specified, the default value is used when inserting a new edge.\nCOMMENT\nThe remarks of a certain property or the edge type itself. The maximum length is 256 bytes. By default, there will be no comments on an edge type.\nTTL_DURATION\nSpecifies the life cycle for the property. The property that exceeds the specified TTL expires. The expiration threshold is the TTL_COL value plus the TTL_DURATION. The default value of TTL_DURATION is 0. It means the data never expires.\nTTL_COL\nSpecifies the property to set a timeout on. The data type of the property must be int or timestamp. An edge type can only specify one field as TTL_COL. For more information on TTL, see TTL options.\nExamples\n CREATE EDGE IF NOT EXISTS follow(degree int);\n# The following example creates an edge type with no properties.\n CREATE EDGE IF NOT EXISTS no_property();\n# The following example creates an edge type with a default value.\n CREATE EDGE IF NOT EXISTS follow_with_default(degree int DEFAULT 20);\n# In the following example, the TTL of the p2 field is set to be 100 seconds.\n CREATE EDGE IF NOT EXISTS e1(p1 string, p2 int, p3 timestamp) \\\n TTL_DURATION = 100, TTL_COL = \"p2\";","url":"11.edge-type-statements/1.create-edge/","type":"doc"},{"title":"DROP EDGE","content":"DROP EDGE drops an edge type with the given name in a graph space.\nAn edge can have only one edge type. After you drop it, the edge CANNOT be accessed. The edge will be deleted in the next compaction.\nThis operation only deletes the Schema data. All the files or directories in the disk will not be deleted directly until the next compaction.\nPrerequisites\nRunning the DROP EDGE statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nBefore you drop an edge type, make sure that the edge type does not have any indexes. Otherwise, the conflict error ([ERROR (-1005)]: Conflict!) will be returned. To drop an index, see DROP INDEX.\nSyntax\nDROP EDGE [IF EXISTS] \nIF NOT EXISTS: Detects if the edge type that you want to drop exists. Only when it exists will it be dropped.\nedge_type_name: Specifies the edge type name that you want to drop. You can drop only one edge type in one statement.\nExample\n CREATE EDGE IF NOT EXISTS e1(p1 string, p2 int);\n DROP EDGE e1;","url":"11.edge-type-statements/2.drop-edge/","type":"doc"},{"title":"ALTER EDGE","content":"ALTER EDGE alters the structure of an edge type with the given name in a graph space. You can add or drop properties, and change the data type of an existing property. You can also set a TTL (Time-To-Live) on a property, or change its TTL duration.\nNotes\nRunning the ALTER EDGE statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nBefore you alter properties for an edge type, make sure that the properties are not indexed. If the properties contain any indexes, the conflict error [ERROR (-1005)]: Conflict! will occur when you ALTER EDGE. For more information on dropping an index, see DROP INDEX.\nThe property name must be unique in an edge type. If you add a property with the same name as an existing property or a dropped property, the operation fails.\nOnly the length of a FIXED_STRING or an INT can be increased.\nOnly the data type conversions from FIXED_STRING to STRING and from FLOAT to DOUBLE are allowed.\nSyntax\nALTER EDGE \n [, alter_definition] ...]\n [ttl_definition [, ttl_definition] ... ]\n [COMMENT = ''];\nalter_definition:\n| ADD (prop_name data_type)\n| DROP (prop_name)\n| CHANGE (prop_name data_type)\nttl_definition:\n TTL_DURATION = ttl_duration, TTL_COL = prop_name\nedge_type_name: Specifies the edge type name that you want to alter. You can alter only one edge type in one statement. Before you alter an edge type, make sure that the edge type exists in the graph space. If the edge type does not exist, an error occurs when you alter it.\nMultiple ADD, DROP, and CHANGE clauses are permitted in a single ALTER EDGE statement, separated by commas.\nWhen a property value is set to NOT NULL using ADD or CHANGE, a default value must be specified for the property, that is, the value of DEFAULT must be specified.\nExample\n CREATE EDGE IF NOT EXISTS e1(p1 string, p2 int);\n ALTER EDGE e1 ADD (p3 int, p4 string);\n ALTER EDGE e1 TTL_DURATION = 2, TTL_COL = \"p2\";\n ALTER EDGE e1 COMMENT = 'edge1';\nImplementation of the operation\nTrying to use a newly altered edge type may fail because the alteration of the edge type is implemented asynchronously. To make sure the follow-up operations work as expected, Wait for two heartbeat cycles, i.e., 20 seconds.\nTo change the heartbeat interval, modify the heartbeat_interval_secs parameter in the configuration files for all services.","url":"11.edge-type-statements/3.alter-edge/","type":"doc"},{"title":"SHOW EDGES","content":"SHOW EDGES shows all edge types in the current graph space.\nYou do not need any privileges for the graph space to run the SHOW EDGES statement. But the returned results are different based on role privileges.\nSyntax\nSHOW EDGES;\nExample\n SHOW EDGES;","url":"11.edge-type-statements/4.show-edges/","type":"doc"},{"title":"DESCRIBE EDGE","content":"DESCRIBE EDGE returns the information about an edge type with the given name in a graph space, such as field names, data type, and so on.\nPrerequisites\nRunning the DESCRIBE EDGE statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nSyntax\nDESC[RIBE] EDGE \nYou can use DESC instead of DESCRIBE for short.\nExample\n DESCRIBE EDGE follow;","url":"11.edge-type-statements/5.describe-edge/","type":"doc"},{"title":"INSERT VERTEX","content":"The INSERT VERTEX statement inserts one or more vertices into a graph space in NebulaGraph.\nPrerequisites\nRunning the INSERT VERTEX statement requires some privileges for the graph space. Otherwise, NebulaGraph throws an error.\nSyntax\nINSERT VERTEX [IF NOT EXISTS] [tag_props, [tag_props] ...]\nVALUES VID: ([prop_value_list])\ntag_props:\n tag_name ([prop_name_list])\nprop_name_list:\n [prop_name [, prop_name] ...]\nprop_value_list:\n [prop_value [, prop_value] ...] \nIF NOT EXISTS detects if the VID that you want to insert exists. If it does not exist, a new one will be inserted.\ntag_name denotes the tag (vertex type), which must be created before INSERT VERTEX. For more information, see CREATE TAG.\nprop_name_list contains the names of the properties on the tag.\nVID is the vertex ID. In NebulaGraph 2.0, string and integer VID types are supported. The VID type is set when a graph space is created. For more information, see CREATE SPACE.\nprop_value_list must provide the property values according to the prop_name_list. When the NOT NULL constraint is set for a given property, an error is returned if no property is given. When the default value for a property is NULL, you can omit to specify the property value. For details, see CREATE TAG.\nExamples are as follows.\nExamples\n# Insert a vertex without tag.\n INSERT VERTEX VALUES \"1\":();\n# The following examples create tag t1 with no property and inserts vertex \"10\" with no property.\n CREATE TAG IF NOT EXISTS t1(); \n INSERT VERTEX t1() VALUES \"10\":(); \n CREATE TAG IF NOT EXISTS t2 (name string, age int); \n INSERT VERTEX t2 (name, age) VALUES \"11\":(\"n1\", 12);\n# In the following example, the insertion fails because \"a13\" is not int.\n INSERT VERTEX t2 (name, age) VALUES \"12\":(\"n1\", \"a13\"); \n# The following example inserts two vertices at one time.\n INSERT VERTEX t2 (name, age) VALUES \"13\":(\"n3\", 12), \"14\":(\"n4\", 8); \n CREATE TAG IF NOT EXISTS t3(p1 int);\n CREATE TAG IF NOT EXISTS t4(p2 string);\n# The following example inserts vertex \"21\" with two tags.\n INSERT VERTEX t3 (p1), t4(p2) VALUES \"21\": (321, \"hello\");\nA vertex can be inserted/written with new values multiple times. Only the last written values can be read.\n# The following examples insert vertex \"11\" with new values for multiple times.\n INSERT VERTEX t2 (name, age) VALUES \"11\":(\"n2\", 13);\n INSERT VERTEX t2 (name, age) VALUES \"11\":(\"n3\", 14);\n INSERT VERTEX t2 (name, age) VALUES \"11\":(\"n4\", 15);\n FETCH PROP ON t2 \"11\" YIELD properties(vertex);\n CREATE TAG IF NOT EXISTS t5(p1 fixed_string(5) NOT NULL, p2 int, p3 int DEFAULT NULL);\n INSERT VERTEX t5(p1, p2, p3) VALUES \"001\":(\"Abe\", 2, 3);\n# In the following example, the insertion fails because the value of p1 cannot be NULL.\n INSERT VERTEX t5(p1, p2, p3) VALUES \"002\":(NULL, 4, 5);\n[ERROR (-1009)]: SemanticError: No schema found for `t5'\n# In the following example, the value of p3 is the default NULL.\n INSERT VERTEX t5(p1, p2) VALUES \"003\":(\"cd\", 5);\n FETCH PROP ON t5 \"003\" YIELD properties(vertex);\n# In the following example, the allowed maximum length of p1 is 5.\n INSERT VERTEX t5(p1, p2) VALUES \"004\":(\"shalalalala\", 4);\n FETCH PROP on t5 \"004\" YIELD properties(vertex);\nIf you insert a vertex that already exists with IF NOT EXISTS, there will be no modification.\n# The following example inserts vertex \"1\".\n INSERT VERTEX t2 (name, age) VALUES \"1\":(\"n2\", 13);\n# Modify vertex \"1\" with IF NOT EXISTS. But there will be no modification as vertex \"1\" already exists.\n INSERT VERTEX IF NOT EXISTS t2 (name, age) VALUES \"1\":(\"n3\", 14);\n FETCH PROP ON t2 \"1\" YIELD properties(vertex);","url":"12.vertex-statements/1.insert-vertex/","type":"doc"},{"title":"DELETE VERTEX","content":"By default, the DELETE VERTEX statement deletes vertices but the incoming and outgoing edges of the vertices.\nThe DELETE VERTEX statement deletes one vertex or multiple vertices at a time. You can use DELETE VERTEX together with pipes. For more information about pipe, see Pipe operator.\nSyntax\nDELETE VERTEX [, ...] [WITH EDGE];\nWITH EDGE: deletes vertices and the related incoming and outgoing edges of the vertices.\nExamples\nThis query deletes the vertex whose ID is \"team1\".\n# Delete the vertex whose VID is `team1` but the related incoming and outgoing edges are not deleted.\n DELETE VERTEX \"team1\";\n# Delete the vertex whose VID is `team1` and the related incoming and outgoing edges.\n DELETE VERTEX \"team1\" WITH EDGE;\nThis query shows that you can use DELETE VERTEX together with pipe to delete vertices.\n GO FROM \"player100\" OVER serve WHERE properties(edge).start_year == \"2021\" YIELD dst(edge) AS id | DELETE VERTEX $-.id;\nProcess of deleting vertices\nOnce NebulaGraph deletes the vertices, all edges (incoming and outgoing edges) of the target vertex will become dangling edges. When NebulaGraph deletes the vertices WITH EDGE, NebulaGraph traverses the incoming and outgoing edges related to the vertices and deletes them all. Then NebulaGraph deletes the vertices.","url":"12.vertex-statements/4.delete-vertex/","type":"doc"},{"title":"UPDATE VERTEX","content":"The UPDATE VERTEX statement updates properties on tags of a vertex.\nIn NebulaGraph, UPDATE VERTEX supports compare-and-set (CAS).\nSyntax\nUPDATE VERTEX ON \nSET \n[WHEN ]\n[YIELD ]\nParameter\nRequired\nDescription\nExample\nON \nYes\nSpecifies the tag of the vertex. The properties to be updated must be on this tag.\nON player\n\nYes\nSpecifies the ID of the vertex to be updated.\n\"player100\"\nSET \nYes\nSpecifies the properties to be updated and how they will be updated.\nSET age = age +1\nWHEN \nNo\nSpecifies the filter conditions. If evaluates to false, the SET clause will not take effect.\nWHEN name == \"Tim\"\nYIELD \nNo\nSpecifies the output format of the statement.\nYIELD name AS Name\nExample\n// This query checks the properties of vertex \"player101\".\n FETCH PROP ON player \"player101\" YIELD properties(vertex);\n// This query updates the age property and returns name and the new age.\n UPDATE VERTEX ON player \"player101\" \\\n SET age = age + 2 \\\n WHEN name == \"Tony Parker\" \\\n YIELD name AS Name, age AS Age;","url":"12.vertex-statements/2.update-vertex/","type":"doc"},{"title":"UPSERT VERTEX","content":"The UPSERT statement is a combination of UPDATE and INSERT. You can use UPSERT VERTEX to update the properties of a vertex if it exists or insert a new vertex if it does not exist.\nThe performance of UPSERT is much lower than that of INSERT because UPSERT is a read-modify-write serialization operation at the partition level.\nSyntax\nUPSERT VERTEX ON \nSET \n[WHEN ]\n[YIELD ]\nParameter\nRequired\nDescription\nExample\nON \nYes\nSpecifies the tag of the vertex. The properties to be updated must be on this tag.\nON player\n\nYes\nSpecifies the ID of the vertex to be updated or inserted.\n\"player100\"\nSET \nYes\nSpecifies the properties to be updated and how they will be updated.\nSET age = age +1\nWHEN \nNo\nSpecifies the filter conditions.\nWHEN name == \"Tim\"\nYIELD \nNo\nSpecifies the output format of the statement.\nYIELD name AS Name\nInsert a vertex if it does not exist\nIf a vertex does not exist, it is created no matter the conditions in the WHEN clause are met or not, and the SET clause always takes effect. The property values of the new vertex depend on:\nHow the SET clause is defined.\nWhether the property has a default value.\nFor example, if:\nThe vertex to be inserted will have properties name and age based on the tag player.\nThe SET clause specifies that age = 30.\nThen the property values in different cases are listed as follows:\nAre WHEN conditions met\nIf properties have default values\nValue of name\nValue of age\nYes\nYes\nThe default value\n30\nYes\nNo\nNULL\n30\nNo\nYes\nThe default value\n30\nNo\nNo\nNULL\n30\nHere are some examples:\n// This query checks if the following three vertices exist. The result \"Empty set\" indicates that the vertices do not exist.\n FETCH PROP ON * \"player666\", \"player667\", \"player668\" YIELD properties(vertex);\nEmpty set\n UPSERT VERTEX ON player \"player666\" \\\n SET age = 30 \\\n WHEN name == \"Joe\" \\\n YIELD name AS Name, age AS Age;\n UPSERT VERTEX ON player \"player666\" \\\n SET age = 31 \\\n WHEN name == \"Joe\" \\\n YIELD name AS Name, age AS Age;\n UPSERT VERTEX ON player \"player667\" \\\n SET age = 31 \\\n YIELD name AS Name, age AS Age;\n UPSERT VERTEX ON player \"player668\" \\\n SET name = \"Amber\", age = age + 1 \\\n YIELD name AS Name, age AS Age;\nIn the last query of the preceding examples, since age has no default value, when the vertex is created, age is NULL, and age = age + 1 does not take effect. But if age has a default value, age = age + 1 will take effect. For example:\n CREATE TAG IF NOT EXISTS player_with_default(name string, age int DEFAULT 20);\nExecution succeeded\n UPSERT VERTEX ON player_with_default \"player101\" \\\n SET age = age + 1 \\\n YIELD name AS Name, age AS Age;\nUpdate a vertex if it exists\nIf the vertex exists and the WHEN conditions are met, the vertex is updated.\n FETCH PROP ON player \"player101\" YIELD properties(vertex);\n UPSERT VERTEX ON player \"player101\" \\\n SET age = age + 2 \\\n WHEN name == \"Tony Parker\" \\\n YIELD name AS Name, age AS Age;\nIf the vertex exists and the WHEN conditions are not met, the update does not take effect.\n FETCH PROP ON player \"player101\" YIELD properties(vertex);\n UPSERT VERTEX ON player \"player101\" \\\n SET age = age + 2 \\\n WHEN name == \"Someone else\" \\\n YIELD name AS Name, age AS Age;","url":"12.vertex-statements/3.upsert-vertex/","type":"doc"},{"title":"INSERT EDGE","content":"The INSERT EDGE statement inserts an edge or multiple edges into a graph space from a source vertex (given by src_vid) to a destination vertex (given by dst_vid) with a specific rank in NebulaGraph.\nWhen inserting an edge that already exists, INSERT EDGE overrides the edge.\nSyntax\nINSERT EDGE [IF NOT EXISTS] ( ) VALUES \n -> [@] : ( )\n[, -> [@] : ( ), ...];\n ::=\n [ [, ] ...]\n ::=\n [ [, ] ...]\nIF NOT EXISTS detects if the edge that you want to insert exists. If it does not exist, a new one will be inserted.\n denotes the edge type, which must be created before INSERT EDGE. Only one edge type can be specified in this statement.\n is the property name list in the given .\nsrc_vid is the VID of the source vertex. It specifies the start of an edge.\ndst_vid is the VID of the destination vertex. It specifies the end of an edge.\nrank is optional. It specifies the edge rank of the same edge type. The data type is int. If not specified, the default value is 0. You can insert many edges with the same edge type, source vertex, and destination vertex by using different rank values.\n must provide the value list according to . If the property values do not match the data type in the edge type, an error is returned. When the NOT NULL constraint is set for a given property, an error is returned if no property is given. When the default value for a property is NULL, you can omit to specify the property value. For details, see CREATE EDGE.\nExamples\n# The following example creates edge type e1 with no property and inserts an edge from vertex \"10\" to vertex \"11\" with no property.\n CREATE EDGE IF NOT EXISTS e1(); \n INSERT EDGE e1 () VALUES \"10\"->\"11\":(); \n# The following example inserts an edge from vertex \"10\" to vertex \"11\" with no property. The edge rank is 1.\n INSERT EDGE e1 () VALUES \"10\"->\"11\"@1:(); \n CREATE EDGE IF NOT EXISTS e2 (name string, age int); \n INSERT EDGE e2 (name, age) VALUES \"11\"->\"13\":(\"n1\", 1);\n# The following example creates edge type e2 with two properties.\n INSERT EDGE e2 (name, age) VALUES \\\n \"12\"->\"13\":(\"n1\", 1), \"13\"->\"14\":(\"n2\", 2); \n# In the following example, the insertion fails because \"a13\" is not int.\n INSERT EDGE e2 (name, age) VALUES \"11\"->\"13\":(\"n1\", \"a13\");\nAn edge can be inserted/written with property values multiple times. Only the last written values can be read.\nThe following examples insert edge e2 with the new values for multiple times.\n INSERT EDGE e2 (name, age) VALUES \"11\"->\"13\":(\"n1\", 12);\n INSERT EDGE e2 (name, age) VALUES \"11\"->\"13\":(\"n1\", 13);\n INSERT EDGE e2 (name, age) VALUES \"11\"->\"13\":(\"n1\", 14);\n FETCH PROP ON e2 \"11\"->\"13\" YIELD edge AS e;\nIf you insert an edge that already exists with IF NOT EXISTS, there will be no modification.\n# The following example inserts edge e2 from vertex \"14\" to vertex \"15\".\n INSERT EDGE e2 (name, age) VALUES \"14\"->\"15\"@1:(\"n1\", 12);\n# The following example alters the edge with IF NOT EXISTS. But there will be no alteration because edge e2 already exists.\n INSERT EDGE IF NOT EXISTS e2 (name, age) VALUES \"14\"->\"15\"@1:(\"n2\", 13);\n FETCH PROP ON e2 \"14\"->\"15\"@1 YIELD edge AS e;","url":"13.edge-statements/1.insert-edge/","type":"doc"},{"title":"DELETE EDGE","content":"The DELETE EDGE statement deletes one edge or multiple edges at a time. You can use DELETE EDGE together with pipe operators. For more information, see PIPE OPERATORS.\nTo delete all the outgoing edges for a vertex, please delete the vertex. For more information, see DELETE VERTEX.\nSyntax\nDELETE EDGE -> [@] [, -> [@] ...]\nExamples\n DELETE EDGE serve \"player100\" -> \"team204\"@0;\nThe following example shows that you can use DELETE EDGE together with pipe operators to delete edges that meet the conditions.\n GO FROM \"player100\" OVER follow \\\n WHERE dst(edge) == \"player101\" \\\n YIELD src(edge) AS src, dst(edge) AS dst, rank(edge) AS rank \\\n | DELETE EDGE follow $-.src->$-.dst @ $-.rank;","url":"13.edge-statements/4.delete-edge/","type":"doc"},{"title":"UPDATE EDGE","content":"The UPDATE EDGE statement updates properties on an edge.\nIn NebulaGraph, UPDATE EDGE supports compare-and-swap (CAS).\nSyntax\nUPDATE EDGE ON \n -> [@]\nSET \n[WHEN ]\n[YIELD ]\nParameter\nRequired\nDescription\nExample\nON \nYes\nSpecifies the edge type. The properties to be updated must be on this edge type.\nON serve\n\nYes\nSpecifies the source vertex ID of the edge.\n\"player100\"\n\nYes\nSpecifies the destination vertex ID of the edge.\n\"team204\"\n\nNo\nSpecifies the rank of the edge. The data type is int.\n10\nSET \nYes\nSpecifies the properties to be updated and how they will be updated.\nSET start_year = start_year +1\nWHEN \nNo\nSpecifies the filter conditions. If evaluates to false, the SET clause does not take effect.\nWHEN end_year < 2010\nYIELD \nNo\nSpecifies the output format of the statement.\nYIELD start_year AS Start_Year\nExample\nThe following example checks the properties of the edge with the GO statement.\n GO FROM \"player100\" \\\n OVER serve \\\n YIELD properties(edge).start_year, properties(edge).end_year;\nThe following example updates the start_year property and returns the end_year and the new start_year.\n UPDATE EDGE on serve \"player100\" -> \"team204\"@0 \\\n SET start_year = start_year + 1 \\\n WHEN end_year > 2010 \\\n YIELD start_year, end_year;","url":"13.edge-statements/2.update-edge/","type":"doc"},{"title":"UPSERT EDGE","content":"The UPSERT statement is a combination of UPDATE and INSERT. You can use UPSERT EDGE to update the properties of an edge if it exists or insert a new edge if it does not exist.\nThe performance of UPSERT is much lower than that of INSERT because UPSERT is a read-modify-write serialization operation at the partition level.\nSyntax\nUPSERT EDGE ON \n -> [@rank]\nSET \n[WHEN ]\n[YIELD ]\nParameter\nRequired\nDescription\nExample\nON \nYes\nSpecifies the edge type. The properties to be updated must be on this edge type.\nON serve\n\nYes\nSpecifies the source vertex ID of the edge.\n\"player100\"\n\nYes\nSpecifies the destination vertex ID of the edge.\n\"team204\"\n\nNo\nSpecifies the rank of the edge.\n10\nSET \nYes\nSpecifies the properties to be updated and how they will be updated.\nSET start_year = start_year +1\nWHEN \nNo\nSpecifies the filter conditions.\nWHEN end_year < 2010\nYIELD \nNo\nSpecifies the output format of the statement.\nYIELD start_year AS Start_Year\nInsert an edge if it does not exist\nIf an edge does not exist, it is created no matter the conditions in the WHEN clause are met or not, and the SET clause takes effect. The property values of the new edge depend on:\nHow the SET clause is defined.\nWhether the property has a default value.\nFor example, if:\nThe edge to be inserted will have properties start_year and end_year based on the edge type serve.\nThe SET clause specifies that end_year = 2021.\nThen the property values in different cases are listed as follows:\nAre WHEN conditions met\nIf properties have default values\nValue of start_year\nValue of end_year\nYes\nYes\nThe default value\n2021\nYes\nNo\nNULL\n2021\nNo\nYes\nThe default value\n2021\nNo\nNo\nNULL\n2021\nHere are some examples:\n// This example checks if the following three vertices have any outgoing serve edge. The result \"Empty set\" indicates that such an edge does not exist.\n GO FROM \"player666\", \"player667\", \"player668\" \\\n OVER serve \\\n YIELD properties(edge).start_year, properties(edge).end_year;\nEmpty set\n UPSERT EDGE on serve \\\n \"player666\" -> \"team200\"@0 \\\n SET end_year = 2021 \\\n WHEN end_year == 2010 \\\n YIELD start_year, end_year;\n UPSERT EDGE on serve \\\n \"player666\" -> \"team200\"@0 \\\n SET end_year = 2022 \\\n WHEN end_year == 2010 \\\n YIELD start_year, end_year;\n UPSERT EDGE on serve \\\n \"player667\" -> \"team200\"@0 \\\n SET end_year = 2022 \\\n YIELD start_year, end_year;\n UPSERT EDGE on serve \\\n \"player668\" -> \"team200\"@0 \\\n SET start_year = 2000, end_year = end_year + 1 \\\n YIELD start_year, end_year;\nIn the last query of the preceding example, since end_year has no default value, when the edge is created, end_year is NULL, and end_year = end_year + 1 does not take effect. But if end_year has a default value, end_year = end_year + 1 will take effect. For example:\n CREATE EDGE IF NOT EXISTS serve_with_default(start_year int, end_year int DEFAULT 2010);\nExecution succeeded\n UPSERT EDGE on serve_with_default \\\n \"player668\" -> \"team200\" \\\n SET end_year = end_year + 1 \\\n YIELD start_year, end_year;\nUpdate an edge if it exists\nIf the edge exists and the WHEN conditions are met, the edge is updated.\n MATCH (v:player{name:\"Ben Simmons\"})-[e:serve]-(v2) \\\n RETURN e;\n UPSERT EDGE on serve \\\n \"player149\" -> \"team219\" \\\n SET end_year = end_year + 1 \\\n WHEN start_year == 2016 \\\n YIELD start_year, end_year;\nIf the edge exists and the WHEN conditions are not met, the update does not take effect.\n MATCH (v:player{name:\"Ben Simmons\"})-[e:serve]-(v2) \\\n RETURN e;\n UPSERT EDGE on serve \\\n \"player149\" -> \"team219\" \\\n SET end_year = end_year + 1 \\\n WHEN start_year != 2016 \\\n YIELD start_year, end_year;","url":"13.edge-statements/3.upsert-edge/","type":"doc"},{"title":"Index overview","content":"Indexes are built to fast process graph queries. Nebula Graph supports two kinds of indexes: native indexes and full-text indexes. This topic introduces the index types and helps choose the right index.\nUsage Instructions\nIndexes can improve query performance but may reduce write performance.\nAn index is a prerequisite for locating data when executing a LOOKUPstatement. If there is no index, an error will be reported when executing the LOOKUP statement.\nWhen using an index, NebulaGraph will automatically select the most optimal index.\nIndexes with high selectivity, that is, when the ratio of the number of records with unique values in the index column to the total number of records is high (for example, the ratio for ID numbers is 1), can significantly improve query performance. For indexes with low selectivity (such as country), query performance might not experience a substantial improvement.\nNative indexes\nNative indexes allow querying data based on a given property. Features are as follows.\nThere are two kinds of native indexes: tag index and edge type index.\nNative indexes must be updated manually. You can use the REBUILD INDEX statement to update native indexes.\nNative indexes support indexing multiple properties on a tag or an edge type (composite indexes), but do not support indexing across multiple tags or edge types.\nOperations on native indexes\nCREATE INDEX\nSHOW CREATE INDEX\nSHOW INDEXES\nDESCRIBE INDEX\nREBUILD INDEX\nSHOW INDEX STATUS\nDROP INDEX\nLOOKUP\nMATCH\nGeography index\nFull-text indexes\nFull-text indexes are used to do prefix, wildcard, regexp, and fuzzy search on a string property. Features are as follows.\nFull-text indexes allow indexing just one property.\nFull-text indexes do not support logical operations such as AND, OR, and NOT.\nNull values\nIndexes do not support indexing null values.\nRange queries\nIn addition to querying single results from native indexes, you can also do range queries. Not all the native indexes support range queries. You can only do range searches for numeric, date, and time type properties.","url":"14.native-index-statements/","type":"doc"},{"title":"CREATE INDEX","content":"Prerequisites\nBefore you create an index, make sure that the relative tag or edge type is created. For how to create tags or edge types, see CREATE TAG and CREATE EDGE.\nFor how to create full-text indexes, see Deploy full-text index.\nMust-read for using indexes\nThe concept and using restrictions of indexes are comparatively complex. Before you use indexes, you must read the following sections carefully.\nYou can use CREATE INDEX to add native indexes for the existing tags, edge types, or properties. They are usually called as tag indexes, edge type indexes, and property indexes.\nTag indexes and edge type indexes apply to queries related to the tag and the edge type, but do not apply to queries that are based on certain properties on the tag. For example, you can use LOOKUP to retrieve all the vertices with the tag player.\nProperty indexes apply to property-based queries. For example, you can use the age property to retrieve the VID of all vertices that meet age == 19.\nIf a property index i_TA is created for the property A of the tag T and i_T for the tag T, the indexes can be replaced as follows (the same for edge type indexes):\nThe query engine can use i_TA to replace i_T.\nIn the MATCH and LOOKUP statements, i_T may replace i_TA for querying properties.\nAlthough the same results can be obtained by using alternative indexes for queries, the query performance varies according to the selected index.\nSteps\nIf you must use indexes, we suggest that you:\nImport the data into NebulaGraph.\nCreate indexes.\nRebuild indexes.\nAfter the index is created and the data is imported, you can use LOOKUP or MATCH to retrieve the data. You do not need to specify which indexes to use in a query, NebulaGraph figures that out by itself.\nSyntax\nCREATE {TAG | EDGE} INDEX [IF NOT EXISTS] ON { | } ([]) [COMMENT ''];\nParameter\nDescription\nTAG | EDGE\nSpecifies the index type that you want to create.\nIF NOT EXISTS\nDetects if the index that you want to create exists. If it does not exist, a new one will be created.\n\n1. The name of the index. It must be unique in a graph space. A recommended way of naming is i_tagName_propName. 2. By default, the name only supports 1-4 byte UTF-8 encoded characters, including English letters (case sensitive), numbers, Chinese characters, etc. However, it cannot include special characters other than the underscore (_), and cannot start with a number.3. To use special characters, reserved keywords, or start with a number, quote the entire name with backticks (`) and do not include periods (.) within the pair of backticks (`). For more information, see Keywords and reserved words.Note: If you name an index in Chinese and encounter a SyntaxError, you need to quote the Chinese characters with backticks (`).\n | \nSpecifies the name of the tag or edge associated with the index.\n\nTo index a variable-length string property, you must use prop_name(length) to specify the index length, and the maximum index length is 256. To index a tag or an edge type, ignore the prop_name_list.\nCOMMENT\nThe remarks of the index. The maximum length is 256 bytes. By default, there will be no comments on an index.\nCreate tag/edge type indexes\n CREATE TAG INDEX IF NOT EXISTS player_index on player();\n CREATE EDGE INDEX IF NOT EXISTS follow_index on follow();\nAfter indexing a tag or an edge type, you can use the LOOKUP statement to retrieve the VID of all vertices with the tag, or the source vertex ID, destination vertex ID, and ranks of all edges with the edge type. For more information, see LOOKUP.\nCreate single-property indexes\n CREATE TAG INDEX IF NOT EXISTS player_index_0 on player(name(10));\nThe preceding example creates an index for the name property on all vertices carrying the player tag. This example creates an index using the first 10 characters of the name property.\n# To index a variable-length string property, you need to specify the index length.\n CREATE TAG IF NOT EXISTS var_string(p1 string);\n CREATE TAG INDEX IF NOT EXISTS var ON var_string(p1(10));\n# To index a fixed-length string property, you do not need to specify the index length.\n CREATE TAG IF NOT EXISTS fix_string(p1 FIXED_STRING(10));\n CREATE TAG INDEX IF NOT EXISTS fix ON fix_string(p1);\n CREATE EDGE INDEX IF NOT EXISTS follow_index_0 on follow(degree);\nCreate composite property indexes\nAn index on multiple properties on a tag (or an edge type) is called a composite property index.\n CREATE TAG INDEX IF NOT EXISTS player_index_1 on player(name(10), age);","url":"14.native-index-statements/1.create-native-index/","type":"doc"},{"title":"SHOW INDEXES","content":"SHOW INDEXES shows the defined tag or edge type indexes names in the current graph space.\nSyntax\nSHOW {TAG | EDGE} INDEXES\nExamples\n SHOW TAG INDEXES;\n SHOW EDGE INDEXES;","url":"14.native-index-statements/2.show-native-indexes/","type":"doc"},{"title":"SHOW CREATE INDEX","content":"SHOW CREATE INDEX shows the statement used when creating a tag or an edge type. It contains detailed information about the index, such as its associated properties.\nSyntax\nSHOW CREATE {TAG | EDGE} INDEX ;\nExamples\nYou can run SHOW TAG INDEXES to list all tag indexes, and then use SHOW CREATE TAG INDEX to show the information about the creation of the specified index.\n SHOW TAG INDEXES;\n SHOW CREATE TAG INDEX player_index_1;\nEdge indexes can be queried through a similar approach.\n SHOW EDGE INDEXES;\n SHOW CREATE EDGE INDEX follow_index;","url":"14.native-index-statements/2.1.show-create-index/","type":"doc"},{"title":"DESCRIBE INDEX","content":"DESCRIBE INDEX can get the information about the index with a given name, including the property name (Field) and the property type (Type) of the index.\nSyntax\nDESCRIBE {TAG | EDGE} INDEX ;\nExamples\n DESCRIBE TAG INDEX player_index_0;\n DESCRIBE TAG INDEX player_index_1;","url":"14.native-index-statements/3.describe-native-index/","type":"doc"},{"title":"REBUILD INDEX","content":"You can use REBUILD INDEX to rebuild the created tag or edge type index. For details on how to create an index, see CREATE INDEX.\nSyntax\nREBUILD {TAG | EDGE} INDEX [];\n::=\n [index_name [, index_name] ...]\nMultiple indexes are permitted in a single REBUILD statement, separated by commas. When the index name is not specified, all tag or edge indexes are rebuilt.\nAfter the rebuilding is complete, you can use the SHOW {TAG | EDGE} INDEX STATUS command to check if the index is successfully rebuilt. For details on index status, see SHOW INDEX STATUS.\nExamples\n CREATE TAG IF NOT EXISTS person(name string, age int, gender string, email string);\n CREATE TAG INDEX IF NOT EXISTS single_person_index ON person(name(10));\n# The following example rebuilds an index and returns the job ID.\n REBUILD TAG INDEX single_person_index;\n# The following example checks the index status.\n SHOW TAG INDEX STATUS;\n# You can also use \"SHOW JOB \" to check if the rebuilding process is complete.\n SHOW JOB 31;\nNebulaGraph creates a job to rebuild the index. The job ID is displayed in the preceding return message. To check if the rebuilding process is complete, use the SHOW JOB statement. For more information, see SHOW JOB.","url":"14.native-index-statements/4.rebuild-native-index/","type":"doc"},{"title":"SHOW INDEX STATUS","content":"SHOW INDEX STATUS returns the name of the created tag or edge type index and its status of job.\nThe status of rebuilding indexes includes:\nQUEUE: The job is in a queue.\nRUNNING: The job is running.\nFINISHED: The job is finished.\nFAILED: The job has failed.\nSTOPPED: The job has stopped.\nINVALID: The job is invalid.\nSyntax\nSHOW {TAG | EDGE} INDEX STATUS;\nExample\n SHOW TAG INDEX STATUS;","url":"14.native-index-statements/5.show-native-index-status/","type":"doc"},{"title":"DROP INDEX","content":"DROP INDEX removes an existing index from the current graph space.\nPrerequisite\nRunning the DROP INDEX statement requires some privileges of DROP TAG INDEX and DROP EDGE INDEX in the given graph space. Otherwise, NebulaGraph throws an error.\nSyntax\nDROP {TAG | EDGE} INDEX [IF EXISTS] ;\nIF EXISTS: Detects whether the index that you want to drop exists. If it exists, it will be dropped.\nExample\n DROP TAG INDEX player_index_0;","url":"14.native-index-statements/6.drop-native-index/","type":"doc"},{"title":"Full-text index restrictions","content":"This topic introduces the restrictions for full-text indexes. Please read the restrictions very carefully before using the full-text indexes.\nFor now, full-text search has the following limitations:\nCurrently, full-text search supports LOOKUP statements only.\nThe full-text index name can contain only numbers, lowercase letters, and underscores.\nThe names of full-text indexes within different graph spaces cannot be duplicated.\nThe query returns 10 records by default. You can use the LIMIT clause to return more records, up to 10,000. You can modify the ElasticSearch parameters to adjust the maximum number of records returned.\nIf there is a full-text index on the tag/edge type, the tag/edge type cannot be deleted or modified.\nThe type of properties must be STRING or FIXED_STRING.\nFull-text index can not be applied to search multiple tags/edge types.\nFull-text index can not search properties with value NULL.\nAltering Elasticsearch indexes is not supported at this time.\nModifying the analyzer is not supported. You have to delete the index data and then specify the analyzer when you rebuild the index.\nMake sure that you start the Elasticsearch cluster and Nebula Graph at the same time. If not, the data writing on the Elasticsearch cluster can be incomplete.\nIt may take a while for Elasticsearch to create indexes. If Nebula Graph warns no index is found, you can check the status of the indexing task.\nNebulaGraph clusters deployed with K8s do not have native support for the full-text search feature. However, you can manually deploy the feature yourself.","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/6.deploy-text-based-index/1.text-based-index-restrictions/","type":"doc"},{"title":"Deploy full-text index","content":"Nebula Graph full-text indexes are powered by Elasticsearch. This means that you can use Elasticsearch full-text query language to retrieve what you want. Full-text indexes are managed through built-in procedures. They can be created only for variable STRING and FIXED_STRING properties when the listener cluster and the Elasticsearch cluster are deployed.\nPrecaution\nBefore you start using the full-text index, please make sure that you know the restrictions.\nDeploy Elasticsearch cluster\nTo deploy an Elasticsearch cluster, see Kubernetes Elasticsearch deployment or Elasticsearch installation.\nYou can configure the Elasticsearch to meet your business needs. To customize the Elasticsearch, see Elasticsearch Document.\nSign in to the text search clients\nWhen the Elasticsearch cluster is deployed, use the SIGN IN statement to sign in to the Elasticsearch clients. Multiple elastic_ip:port pairs are separated with commas. You must use the IPs and the port number in the configuration file for the Elasticsearch.\nSyntax\nSIGN IN TEXT SERVICE (, {HTTP | HTTPS} [,\"\", \"\"]) [, (, ...)];\nExample\n SIGN IN TEXT SERVICE (192.168.8.100:9200, HTTP);\nShow text search clients\nThe SHOW TEXT SEARCH CLIENTS statement can list the text search clients.\nSyntax\nSHOW TEXT SEARCH CLIENTS;\nExample\n SHOW TEXT SEARCH CLIENTS;\nSign out to the text search clients\nThe SIGN OUT TEXT SERVICE statement can sign out all the text search clients.\nSyntax\nSIGN OUT TEXT SERVICE;\nExample\n SIGN OUT TEXT SERVICE;","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/6.deploy-text-based-index/2.deploy-es/","type":"doc"},{"title":"Deploy Raft Listener for NebulaGraph Storage service","content":"Full-text index data is written to the Elasticsearch cluster asynchronously. The Raft Listener (Listener for short) is a separate process that fetches data from the Storage Service and writes them into the Elasticsearch cluster.\nPrerequisites\nYou have read and fully understood the restrictions for using full-text indexes.\nYou have deployed a NebulaGraph cluster.\nYou have deployed a Elasticsearch cluster.\nYou have prepared one or multiple servers to run one or multiple raft listeners.\nPrecautions\nThe Storage Service that you want to run as the Listener must have the same or later release with all the other Nebula Graph services in the cluster.\nFor now, you can only add all Listeners to a graph space once and for all. Trying to add a new Listener to a graph space that already has a Listener will fail. To add all Listeners, set them in one statement.\nDeployment process\nStep 1: Install the Listener service\nThe Listener service uses the same binary as the storaged service. However, the configuration files are different and the processes use different ports. You can install NebulaGraph on all servers that need to deploy a Listener, but only the storaged service can be used. For details, see Install NebulaGraph by RPM or DEB Package.\nStep 2: Prepare the configuration file for the Listener\nIn the etc directory, remove the suffix from nebula-storaged-listener.conf.default or nebula-storaged-listener.conf.production to nebula-storaged-listener.conf, and then modify the configuration content.\nMost configurations are the same as the configurations of Storage Service. This topic only introduces the differences.\nName\nDefault value\nDescription\ndaemonize\ntrue\nWhen set to true, the process is a daemon process.\npid_file\npids/nebula-metad-listener.pid\nThe file that records the process ID.\nmeta_server_addrs\nIP (or hostname) and ports of all Meta services. Multiple Meta services are separated by commas.\nlocal_ip\nThe local IP (or hostname) of the Listener service. Use real IP addresses instead of domain names or loopback IP addresses such as 127.0.0.1.\nport\nThe listening port of the RPC daemon of the Listener service.\nheartbeat_interval_secs\n10\nThe heartbeat interval of the Meta service. The unit is second (s).\nlistener_path\ndata/listener\nThe WAL directory of the Listener. Only one directory is allowed.\ndata_path\ndata\nFor compatibility reasons, this parameter can be ignored. Fill in the default value data.\npart_man_type\nmemory\nThe type of the part manager. Optional values ​​are memory and meta.\nrocksdb_batch_size\n4096\nThe default reserved bytes for batch operations.\nrocksdb_block_cache\nThe default block cache size of BlockBasedTable. The unit is Megabyte (MB).\nengine_type\nrocksdb\nThe type of the Storage engine, such as rocksdb, memory, etc.\npart_type\nsimple\nThe type of the part, such as simple, consensus, etc.\nStep 3: Start Listeners\nTo initiate the Listener, navigate to the installation path of the desired cluster and execute the following command:\n./bin/nebula-storaged --flagfile etc/nebula-storaged-listener.conf\nStep 4: Add Listeners to NebulaGraph\nConnect to NebulaGraph and run USE to enter the graph space that you want to create full-text indexes for. Then run the following statement to add a Listener into NebulaGraph.\nADD LISTENER ELASTICSEARCH [,, ...]\nShow Listeners\nRun the SHOW LISTENER statement to list all Listeners.\nExample\n SHOW LISTENER;\nRemove Listeners\nRun the REMOVE LISTENER ELASTICSEARCH statement to remove all Listeners in a graph space.\nExample\n REMOVE LISTENER ELASTICSEARCH;","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/6.deploy-text-based-index/3.deploy-listener/","type":"doc"},{"title":"Full-text indexes","content":"Full-text indexes are used to do prefix, wildcard, regexp, and fuzzy search on a string property.\nYou can use the WHERE clause to specify the search strings in LOOKUP statements.\nPrerequisite\nBefore using the full-text index, make sure that you have deployed a Elasticsearch cluster and a Listener cluster. For more information, see Deploy Elasticsearch and Deploy Listener.\nPrecaution\nBefore using the full-text index, make sure that you know the restrictions.\nFull Text Queries\nFull-text queries enable you to search for parsed text fields, using a parser with strict syntax to return content based on the query string provided. For details, see Query string query.\nSyntax\nCreate full-text indexes\nCREATE FULLTEXT {TAG | EDGE} INDEX ON { | } ( [,]...) [ANALYZER=\"\"];\nComposite indexes with multiple properties are supported when creating full-text indexes.\n is the name of the analyzer. The default value is standard. To use other analyzers (e.g. IK Analysis), you need to make sure that the corresponding analyzer is installed in Elasticsearch in advance.\nShow full-text indexes\nSHOW FULLTEXT INDEXES;\nRebuild full-text indexes\nREBUILD FULLTEXT INDEX;\nDrop full-text indexes\nDROP FULLTEXT INDEX ;\nUse query options\nLOOKUP ON { | } WHERE ES_QUERY(, \"\") YIELD [| LIMIT [,] ];\n\n [AS ] [, [AS ] ...] [, id(vertex) [AS ]] [, score() AS ]\nindex_name: The name of the full-text index.\ntext: Search conditions. The where can only be followed by the ES_QUERY, and all judgment conditions must be written in the text. For supported syntax, see Query string syntax.\nscore(): The score calculated by doing N degree expansion for the eligible vertices. The default value is 1.0. The higher the score, the higher the degree of match. The return value is sorted by default from highest to lowest score. For details, see Search and Scoring in Lucene.\nExamples\n// This example creates the graph space.\n CREATE SPACE IF NOT EXISTS basketballplayer (partition_num=3,replica_factor=1, vid_type=fixed_string(30));\n// This example signs in the text service.\n SIGN IN TEXT SERVICE (192.168.8.100:9200, HTTP);\n// This example checks the text service status.\n SHOW TEXT SEARCH CLIENTS;\n// This example switches the graph space.\n USE basketballplayer;\n// This example adds the listener to the NebulaGraph cluster.\n ADD LISTENER ELASTICSEARCH 192.168.8.100:9789;\n// This example checks the listener status. When the status is `Online`, the listener is ready.\n SHOW LISTENER;\n// This example creates the tag.\n CREATE TAG IF NOT EXISTS player(name string, city string);\n// This example creates a single-attribute full-text index.\n CREATE FULLTEXT TAG INDEX fulltext_index_1 ON player(name) ANALYZER=\"standard\";\n// This example creates a multi-attribute full-text indexe.\n CREATE FULLTEXT TAG INDEX fulltext_index_2 ON player(name,city) ANALYZER=\"standard\";\n// This example rebuilds the full-text index.\n REBUILD FULLTEXT INDEX;\n// This example shows the full-text index.\n SHOW FULLTEXT INDEXES;\n// This example inserts the test data.\n INSERT VERTEX player(name, city) VALUES \\\n \"Russell Westbrook\": (\"Russell Westbrook\", \"Los Angeles\"), \\\n \"Chris Paul\": (\"Chris Paul\", \"Houston\"),\\\n \"Boris Diaw\": (\"Boris Diaw\", \"Houston\"),\\\n \"David West\": (\"David West\", \"Philadelphia\"),\\\n \"Danny Green\": (\"Danny Green\", \"Philadelphia\"),\\\n \"Tim Duncan\": (\"Tim Duncan\", \"New York\"),\\\n \"James Harden\": (\"James Harden\", \"New York\"),\\\n \"Tony Parker\": (\"Tony Parker\", \"Chicago\"),\\\n \"Aron Baynes\": (\"Aron Baynes\", \"Chicago\"),\\\n \"Ben Simmons\": (\"Ben Simmons\", \"Phoenix\"),\\\n \"Blake Griffin\": (\"Blake Griffin\", \"Phoenix\");\n// These examples run test queries.\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_1,\"Chris\") YIELD id(vertex);\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_1,\"Harden\") YIELD properties(vertex);\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_1,\"Da*\") YIELD properties(vertex);\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_1,\"*b*\") YIELD id(vertex);\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_1,\"*b*\") YIELD id(vertex) | LIMIT 2,3;\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_1,\"*b*\") YIELD id(vertex) | YIELD count(*);\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_1,\"*b*\") YIELD id(vertex), score() AS score;\n// For documents containing a word `b`, its score will be multiplied by a weighting factor of 4, while for documents containing a word `c`, the default weighting factor of 1 is used.\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_1,\"*b*^4 OR *c*\") YIELD id(vertex), score() AS score;\n// When using a multi-attribute full-text index query, the conditions are matched within all properties of the index.\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_2,\"*h*\") YIELD properties(vertex);\n// When using multi-attribute full-text index queries, you can specify different text for different properties for the query.\n LOOKUP ON player WHERE ES_QUERY(fulltext_index_2,\"name:*b* AND city:Houston\") YIELD properties(vertex);\n// Delete single-attribute full-text index.\n DROP FULLTEXT INDEX fulltext_index_1;","url":"15.full-text-index-statements/1.search-with-text-based-index/","type":"doc"},{"title":"EXPLAIN and PROFILE","content":"EXPLAIN helps output the execution plan of an nGQL statement without executing the statement.\nPROFILE executes the statement, then outputs the execution plan as well as the execution profile. You can optimize the queries for better performance according to the execution plan and profile.\nExecution Plan\nThe execution plan is determined by the execution planner in the NebulaGraph query engine.\nThe execution planner processes the parsed nGQL statements into actions. An action is the smallest unit that can be executed. A typical action fetches all neighbors of a given vertex, gets the properties of an edge, and filters vertices or edges based on the given conditions. Each action is assigned to an operator that performs the action.\nFor example, a SHOW TAGS statement is processed into two actions and assigned to a Start operator and a ShowTags operator, while a more complex GO statement may be processed into more than 10 actions and assigned to 10 operators.\nSyntax\nEXPLAINEXPLAIN [format= {\"row\" | \"dot\" | \"tck\"}] ;\nPROFILEPROFILE [format= {\"row\" | \"dot\" | \"tck\"}] ;\nOutput formats\nThe output of an EXPLAIN or a PROFILE statement has three formats, the default row format, the dot format, and the tck format. You can use the format option to modify the output format. Omitting the format option indicates using the default row format.\nThe row format\nThe row format outputs the return message in a table as follows.\nEXPLAIN EXPLAIN format=\"row\" SHOW TAGS;\nExecution succeeded (time spent 327/892 us)\nExecution Plan\n-----+----------+--------------+----------------+----------------------------------------------------------------------\n-----+----------+--------------+----------------+----------------------------------------------------------------------\n-----+----------+--------------+----------------+----------------------------------------------------------------------\n-----+----------+--------------+----------------+----------------------------------------------------------------------\nPROFILE PROFILE format=\"row\" SHOW TAGS;\nGot 2 rows (time spent 2038/2728 us)\nExecution Plan\n-----+----------+--------------+----------------------------------------------------+----------------------------------------------------------------------\n-----+----------+--------------+----------------------------------------------------+----------------------------------------------------------------------\n-----+----------+--------------+----------------------------------------------------+----------------------------------------------------------------------\n-----+----------+--------------+----------------------------------------------------+----------------------------------------------------------------------\nThe descriptions are as follows.\nParameter\nDescription\nid\nThe ID of the operator.\nname\nThe name of the operator.\ndependencies\nThe ID of the operator that the current operator depends on.\nprofiling data\nThe content of the execution profile. ver is the version of the operator. rows shows the number of rows to be output by the operator. execTime shows the execution time of action. totalTime is the sum of the execution time, the system scheduling time, and the queueing time.\noperator info\nThe detailed information of the operator.\nThe dot format\nYou can use the format=\"dot\" option to output the return message in the dot language, and then use Graphviz to generate a graph of the plan.\n EXPLAIN format=\"dot\" SHOW TAGS;\nExecution succeeded (time spent 161/665 us)\nExecution Plan\n--------------------------------------------------------------------------------------------------------------------------------------------- -------------\n plan\n--------------------------------------------------------------------------------------------------------------------------------------------- -------------\n digraph exec_plan {\n rankdir=LR;\n \"ShowTags_0\"[label=\"ShowTags_0|outputVar: \\[\\{\\\"colNames\\\":\\[\\],\\\"name\\\":\\\"__ShowTags_0\\\",\\\"type\\\":\\\"DATASET\\\"\\}\\]\\l|inputVar:\\l\", shape=Mrecord];\n \"Start_2\"->\"ShowTags_0\";\n \"Start_2\"[label=\"Start_2|outputVar: \\[\\{\\\"colNames\\\":\\[\\],\\\"name\\\":\\\"__Start_2\\\",\\\"type\\\":\\\"DATASET\\\"\\}\\]\\l|inputVar: \\l\", shape=Mrecord];\n--------------------------------------------------------------------------------------------------------------------------------------------- -------------\nThe Graphviz graph transformed from the above DOT statement is as follows.\nThe tck format\nThe tck format is similar to a table, but without borders and dividing lines between rows. You can use the results as test cases for unit testing. \nFor information on tck format test cases, see TCK cases.\nEXPLAIN EXPLAIN format=\"tck\" FETCH PROP ON player \"player_1\",\"player_2\",\"player_3\" YIELD properties(vertex).name as name, properties(vertex).age as age;\nExecution succeeded (time spent 261µs/613.718µs)\nExecution Plan (optimize time 28 us)\nWed, 22 Mar 2023 23:15:52 CST\nPROFILE PROFILE format=\"tck\" FETCH PROP ON player \"player_1\",\"player_2\",\"player_3\" YIELD properties(vertex).name as name, properties(vertex).age as age;\nGot 3 rows (time spent 1.474ms/2.19677ms)\nExecution Plan (optimize time 41 us)\nWed, 22 Mar 2023 23:16:13 CST","url":"17.query-tuning-statements/1.explain-and-profile/","type":"doc"},{"title":"Kill queries","content":"KILL QUERY can terminate the query being executed, and is often used to terminate slow queries.\nSyntax\nKILL QUERY (session=, plan=);\nsession_id: The ID of the session.\nplan_id: The ID of the execution plan.\nThe ID of the session and the ID of the execution plan can uniquely determine a query. Both can be obtained through the SHOW QUERIES statement.\nExamples\nThis example executes KILL QUERY in one session to terminate the query in another session.\n KILL QUERY(SESSION=1625553545984255,PLAN=163);\nThe query will be terminated and the following information will be returned.\n[ERROR (-1005)]: ExecutionPlanId[1001] does not exist in current Session.","url":"17.query-tuning-statements/6.kill-query/","type":"doc"},{"title":"Kill sessions","content":"The KILL SESSION command is to terminate running sessions.\nSyntax\nYou can run the KILL SESSION command to terminate one or multiple sessions. The syntax is as follows:\nTo terminate one session\nKILL {SESSION|SESSIONS} \n{SESSION|SESSIONS}: SESSION or SESSIONS, both are supported. \n: Specifies the ID of one session. You can run the SHOW SESSIONS command to view the IDs of sessions.\nTo terminate multiple sessions\nSHOW SESSIONS \n| YIELD $-.SessionId AS sid [WHERE ]\n| KILL {SESSION|SESSIONS} $-.sid\n[WHERE ]：\nOptional, the WHERE clause is used to filter sessions. specifies a session filtering expression, for example, WHERE $-.CreateTime < datetime(\"2022-12-14T18:00:00\"). If the WHERE clause is not specified, all sessions are terminated.\nFiltering conditions in a WHERE clause include: SessionId, UserName, SpaceName, CreateTime, UpdateTime, GraphAddr, Timezone, and ClientIp. You can run the SHOW SESSIONS command to view descriptions of these conditions.\n{SESSION|SESSIONS}: SESSION or SESSIONS, both are supported.\nExamples\nTo terminate one session\n KILL SESSION 1672887983842984 \nTo terminate multiple sessions\nTerminate all sessions whose creation time is less than 2023-01-05T18:00:00.\n SHOW SESSIONS | YIELD $-.SessionId AS sid WHERE $-.CreateTime < datetime(\"2023-01-05T18:00:00\") | KILL SESSIONS $-.sid\nTerminates the two sessions with the earliest creation times.\n SHOW SESSIONS | YIELD $-.SessionId AS sid, $-.CreateTime as CreateTime | ORDER BY $-.CreateTime ASC | LIMIT 2 | KILL SESSIONS $-.sid\nTerminates all sessions created by the username session_user1.\n SHOW SESSIONS | YIELD $-.SessionId as sid WHERE $-.UserName == \"session_user1\" | KILL SESSIONS $-.sid\nTerminate all sessions.\n SHOW SESSIONS | YIELD $-.SessionId as sid | KILL SESSION $-.sid\n// Or\n SHOW SESSIONS | KILL SESSIONS $-.SessionId","url":"17.query-tuning-statements/2.kill-session/","type":"doc"},{"title":"Job manager and the JOB statements","content":"The long-term tasks run by the Storage Service are called jobs, such as COMPACT, FLUSH, and STATS. These jobs can be time-consuming if the data amount in the graph space is large. The job manager helps you run, show, stop, and recover jobs.\nSUBMIT JOB BALANCE LEADER\nStarts a job to balance the distribution of all the storage leaders in all graph spaces. It returns the job ID.\nFor example:\n SUBMIT JOB BALANCE LEADER;\nSUBMIT JOB COMPACT\nThe SUBMIT JOB COMPACT statement triggers the long-term RocksDB compact operation in the current graph space.\nFor more information about compact configuration, see Storage Service configuration.\nFor example:\n SUBMIT JOB COMPACT;\nSUBMIT JOB FLUSH\nThe SUBMIT JOB FLUSH statement writes the RocksDB memfile in the memory to the hard disk in the current graph space.\nFor example:\n SUBMIT JOB FLUSH;\nSUBMIT JOB STATS\nThe SUBMIT JOB STATS statement starts a job that makes the statistics of the current graph space. Once this job succeeds, you can use the SHOW STATS statement to list the statistics. For more information, see SHOW STATS.\nFor example:\n SUBMIT JOB STATS;\nSUBMIT JOB DOWNLOAD/INGEST\nThe SUBMIT JOB DOWNLOAD HDFS and SUBMIT JOB INGEST commands are used to import the SST file into NebulaGraph. For detail, see Import data from SST files.\nThe SUBMIT JOB DOWNLOAD HDFS command will download the SST file on the specified HDFS.\nThe SUBMIT JOB INGEST command will import the downloaded SST file into NebulaGraph.\nFor example:\n SUBMIT JOB DOWNLOAD HDFS \"hdfs://192.168.10.100:9000/sst\";\n SUBMIT JOB INGEST;\nSHOW JOB\nThe Meta Service parses a SUBMIT JOB request into multiple tasks and assigns them to the nebula-storaged processes. The SHOW JOB statement shows the information about a specific job and all its tasks in the current graph space.\njob_id is returned when you run the SUBMIT JOB statement.\nFor example:\n SHOW JOB 8;\nThe descriptions are as follows.\nParameter\nDescription\nJob Id(TaskId)\nThe first row shows the job ID and the other rows show the task IDs and the last row shows the total number of job-related tasks.\nCommand(Dest)\nThe first row shows the command executed and the other rows show on which storaged processes the task is running. The last row shows the number of successful tasks related to the job.\nStatus\nShows the status of the job or task. The last row shows the number of failed tasks related to the job. For more information, see Job status.\nStart Time\nShows a timestamp indicating the time when the job or task enters the RUNNING phase. The last row shows the number of ongoing tasks related to the job.\nStop Time\nShows a timestamp indicating the time when the job or task gets FINISHED, FAILED, or STOPPED.\nError Code\nThe error code of job.\nJob status\nThe descriptions are as follows.\nStatus\nDescription\nQUEUE\nThe job or task is waiting in a queue. The Start Time is empty in this phase.\nRUNNING\nThe job or task is running. The Start Time shows the beginning time of this phase.\nFINISHED\nThe job or task is successfully finished. The Stop Time shows the time when the job or task enters this phase.\nFAILED\nThe job or task has failed. The Stop Time shows the time when the job or task enters this phase.\nSTOPPED\nThe job or task is stopped without running. The Stop Time shows the time when the job or task enters this phase.\nREMOVED\nThe job or task is removed.\nThe description of switching the status is described as follows.\nQueue -- running -- finished -- removed\n \\ \\ /\n \\ \\ -- failed -- /\n \\ \\ /\n \\ ---------- stopped -/\nSHOW JOBS\nThe SHOW JOBS statement lists all the unexpired jobs in the current graph space.\nThe default job expiration interval is one week. You can change it by modifying the job_expired_secs parameter of the Meta Service. For how to modify job_expired_secs, see Meta Service configuration.\nFor example:\n SHOW JOBS;\nSTOP JOB\nThe STOP JOB statement stops jobs that are not finished in the current graph space.\nFor example:\n STOP JOB 22;\nRECOVER JOB\nThe RECOVER JOB [] statement re-executes the jobs that status is FAILED or STOPPED in the current graph space and returns the number of recovered jobs. If is not specified, re-execution is performed from the earliest job and the number of jobs that have been recovered is returned.\nFor example:\n RECOVER JOB;\nFAQ\nHow to troubleshoot job problems?\nThe SUBMIT JOB operations use the HTTP port. Please check if the HTTP ports on the machines where the Storage Service is running are working well. You can use the following command to debug.\ncurl \"http://{storaged-ip}:19779/admin?space={space_name}&op=compact\"","url":"4.job-statements/","type":"doc"},{"title":"Prepare resources for compiling, installing, and running NebulaGraph","content":"This topic describes the requirements and suggestions for compiling and installing NebulaGraph, as well as how to estimate the resource you need to reserve for running a NebulaGraph cluster.\nAbout storage devices\nNebulaGraph is designed and implemented for NVMe SSD. All default parameters are optimized for the SSD devices and require extremely high IOPS and low latency.\nDue to the poor IOPS capability and long random seek latency, HDD is not recommended. Users may encounter many problems when using HDD.\nDo not use remote storage devices, such as NAS or SAN. Do not connect an external virtual hard disk based on HDFS or Ceph.\nDo not use RAID.\nUse local SSD devices, or AWS Provisioned IOPS SSD equivalence.\nAbout CPU architecture\nRequirements for compiling the source code\nHardware requirements for compiling NebulaGraph\nItem\nRequirement\nCPU architecture\nx86_64\nMemory\n4 GB\nDisk\n10 GB, SSD\nSupported operating systems for compiling NebulaGraph\nFor now, we can only compile NebulaGraph in the Linux system. We recommend that you use any Linux system with kernel version 4.15 or above.\nSoftware requirements for compiling NebulaGraph\nYou must have the correct version of the software listed below to compile NebulaGraph. If they are not as required or you are not sure, follow the steps in Prepare software for compiling NebulaGraph to get them ready.\nSoftware\nVersion\nNote\nglibc\n2.17 or above\nYou can run ldd --version to check the glibc version.\nmake\nAny stable version\nm4\nAny stable version\ngit\nAny stable version\nwget\nAny stable version\nunzip\nAny stable version\nxz\nAny stable version\nreadline-devel\nAny stable version\nncurses-devel\nAny stable version\nzlib-devel\nAny stable version\ng++\n8.5.0 or above\nYou can run gcc -v to check the gcc version.\ncmake\n3.14.0 or above\nYou can run cmake --version to check the cmake version.\ncurl\nAny stable version\nredhat-lsb-core\nAny stable version\nlibstdc++-static\nAny stable version\nOnly needed in CentOS 8+, RedHat 8+, and Fedora systems.\nlibasan\nAny stable version\nOnly needed in CentOS 8+, RedHat 8+, and Fedora systems.\nbzip2\nAny stable version\nOther third-party software will be automatically downloaded and installed to the build directory at the configure (cmake) stage.\nPrepare software for compiling NebulaGraph\nIf part of the dependencies are missing or the versions does not meet the requirements, manually install them with the following steps. You can skip unnecessary dependencies or steps according to your needs.\nInstall dependencies.\nFor CentOS, RedHat, and Fedora users, run the following commands.$ yum update\n$ yum install -y make \\\n m4 \\\n git \\\n wget \\\n unzip \\\n xz \\\n readline-devel \\\n ncurses-devel \\\n zlib-devel \\\n gcc \\\n gcc-c++ \\\n cmake \\\n curl \\\n redhat-lsb-core \\\n bzip2\n // For CentOS 8+, RedHat 8+, and Fedora, install libstdc++-static and libasan as well\n$ yum install -y libstdc++-static libasan\nFor Debian and Ubuntu users, run the following commands.$ apt-get update\n$ apt-get install -y make \\\n m4 \\\n git \\\n wget \\\n unzip \\\n xz-utils \\\n curl \\\n lsb-core \\\n build-essential \\\n libreadline-dev \\\n ncurses-dev \\\n cmake \\\n bzip2\nCheck if the GCC and cmake on your host are in the right version. See Software requirements for compiling NebulaGraph for the required versions.\n$ g++ --version\n$ cmake --version\nIf your GCC and CMake are in the right versions, then you are all set and you can ignore the subsequent steps. If they are not, select and perform the needed steps as follows.\nIf the CMake version is incorrect, visit the CMake official website to install the required version.\nIf the G++ version is incorrect, visit the G++ official website or follow the instructions below to to install the required version.\nFor CentOS users, run:\nyum install centos-release-scl\nyum install devtoolset-11\nscl enable devtoolset-11 'bash'\nFor Ubuntu users, run:\nadd-apt-repository ppa:ubuntu-toolchain-r/test\napt install gcc-11 g++-11\nRequirements and suggestions for installing NebulaGraph in test environments\nHardware requirements for test environments\nItem\nRequirement\nCPU architecture\nx86_64\nNumber of CPU core\nMemory\n8 GB\nDisk\n100 GB, SSD\nSupported operating systems for test environments\nFor now, we can only install NebulaGraph in the Linux system. To install NebulaGraph in a test environment, we recommend that you use any Linux system with kernel version 3.9 or above.\nSuggested service architecture for test environments\nProcess\nSuggested number\nmetad (the metadata service process)\nstoraged (the storage service process)\n1 or more\ngraphd (the query engine service process)\n1 or more\nFor example, for a single-machine test environment, you can deploy 1 metad, 1 storaged, and 1 graphd processes in the machine.\nFor a more common test environment, such as a cluster of 3 machines (named as A, B, and C), you can deploy NebulaGraph as follows:\nMachine name\nNumber of metad\nNumber of storaged\nNumber of graphd\nNone\nNone\nRequirements and suggestions for installing NebulaGraph in production environments\nHardware requirements for production environments\nItem\nRequirement\nCPU architecture\nx86_64\nNumber of CPU core\n48\nMemory\n256 GB\nDisk\n2 * 1.6 TB, NVMe SSD\nSupported operating systems for production environments\nFor now, we can only install NebulaGraph in the Linux system. To install NebulaGraph in a production environment, we recommend that you use any Linux system with kernel version 3.9 or above.\nUsers can adjust some of the kernel parameters to better accommodate the need for running NebulaGraph. For more information, see kernel configuration.\nSuggested service architecture for production environments\nProcess\nSuggested number\nmetad (the metadata service process)\nstoraged (the storage service process)\n3 or more\ngraphd (the query engine service process)\n3 or more\nEach metad process automatically creates and maintains a replica of the metadata. Usually, you need to deploy three metad processes and only three.\nThe number of storaged processes does not affect the number of graph space replicas.\nUsers can deploy multiple processes on a single machine. For example, on a cluster of 5 machines (named as A, B, C, D, and E), you can deploy NebulaGraph as follows:\nMachine name\nNumber of metad\nNumber of storaged\nNumber of graphd\nNone\nNone\nCapacity requirements for running a NebulaGraph cluster\nUsers can estimate the memory, disk space, and partition number needed for a NebulaGraph cluster of 3 replicas as follows.\nResource\nUnit\nHow to estimate\nDescription\nDisk space for a cluster\nBytes\nthe_sum_of_edge_number_and_vertex_number * average_bytes_of_properties * 7.5 * 120%\nFor more information, see Edge partitioning and storage amplification.\nMemory for a cluster\nBytes\n[the_sum_of_edge_number_and_vertex_number * 16 + the_number_of_RocksDB_instances * (write_buffer_size * max_write_buffer_number) + rocksdb_block_cache] * 120%\nwrite_buffer_size and max_write_buffer_number are RocksDB parameters. For more information, see MemTable. For details about rocksdb_block_cache, see Memory usage in RocksDB.\nNumber of partitions for a graph space\nthe_number_of_disks_in_the_cluster * disk_partition_num_multiplier\ndisk_partition_num_multiplier is an integer between 2 and 20 (both including). Its value depends on the disk performance. Use 20 for SSD and 2 for HDD.\nQuestion 1: Why do I need to multiply by 7.5 in the disk space estimation formula?Answer: On one hand, the data in one single replica takes up about 2.5 times more space than that of the original data file (csv) according to test values. \n On the other hand, indexes take up additional space. Each indexed vertex or edge takes up 16 bytes of memory. The hard disk space occupied by the index can be empirically estimated as the total number of indexed vertices or edges * 50 bytes.\nQuestion 2: Why do we multiply the disk space and memory by 120%?Answer: The extra 20% is for buffer.\nQuestion 3: How to get the number of RocksDB instances?\nAnswer: Each graph space corresponds to one RocksDB instance and each directory in the --data_path item in the etc/nebula-storaged.conf file corresponds to one RocksDB instance. \n That is, the number of RocksDB instances = the number of directories * the number of graph spaces.","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/1.resource-preparations/","type":"doc"},{"title":"Install NebulaGraph by compiling the source code","content":"Installing NebulaGraph from the source code allows you to customize the compiling and installation settings and test the latest features.\nPrerequisites\nUsers have to prepare correct resources described in Prepare resources for compiling, installing, and running NebulaGraph.\nThe host to be installed with NebulaGraph has access to the Internet.\nInstallation steps\nUse Git to clone the source code of NebulaGraph to the host.\n[Recommended] To install NebulaGraph 3.6.0, run the following command.\n$ git clone --branch release-3.6 https://github.com/vesoft-inc/nebula.git\nTo install the latest developing release, run the following command to clone the source code from the master branch.\n$ git clone https://github.com/vesoft-inc/nebula.git\nGo to the nebula/third-party directory, and run the install-third-party.sh script to install the third-party libraries.\n$ cd nebula/third-party\n$ ./install-third-party.sh\nGo back to the nebula directory, create a directory named build, and enter the directory.\n$ cd ..\n$ mkdir build && cd build\nGenerate Makefile with CMake.\n$ cmake -DCMAKE_INSTALL_PREFIX=/usr/local/nebula -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release ..\nCompile NebulaGraph.\nTo speed up the compiling, use the -j option to set a concurrent number N. It should be \\(\\min(\\text{CPU core number},\\frac{\\text{the memory size(GB)}}{2})\\).\n$ make -j{N} # E.g., make -j2\nInstall NebulaGraph.\n$ sudo make install\nUpdate the master branch\nThe source code of the master branch changes frequently. If the corresponding NebulaGraph release is installed, update it in the following steps.\nIn the nebula directory, run git pull upstream master to update the source code.\nIn the nebula/build directory, run make -j{N} and make install again.\nNext to do\nManage NebulaGraph services\nCMake variables\nUsage of CMake variables\n$ cmake -D= ...\nThe following CMake variables can be used at the configure (cmake) stage to adjust the compiling settings.\nCMAKE_INSTALL_PREFIX\nCMAKE_INSTALL_PREFIX specifies the path where the service modules, scripts, configuration files are installed. The default path is /usr/local/nebula.\nENABLE_WERROR\nENABLE_WERROR is ON by default and it makes all warnings into errors. You can set it to OFF if needed.\nENABLE_TESTING\nENABLE_TESTING is ON by default and unit tests are built with the NebulaGraph services. If you just need the service modules, set it to OFF.\nENABLE_ASAN\nENABLE_ASAN is OFF by default and the building of ASan (AddressSanitizer), a memory error detector, is disabled. To enable it, set ENABLE_ASAN to ON. This variable is intended for NebulaGraph developers.\nCMAKE_BUILD_TYPE\nNebulaGraph supports the following building types of MAKE_BUILD_TYPE:\nDebug\nThe default value of CMAKE_BUILD_TYPE. It indicates building NebulaGraph with the debug info but not the optimization options.\nRelease\nIt indicates building NebulaGraph with the optimization options but not the debug info.\nRelWithDebInfo\nIt indicates building NebulaGraph with the optimization options and the debug info.\nMinSizeRel\nIt indicates building NebulaGraph with the optimization options for controlling the code size but not the debug info.\nENABLE_INCLUDE_WHAT_YOU_USE\nENABLE_INCLUDE_WHAT_YOU_USE is OFF by default. When set to ON and include-what-you-use is installed on the system, the system reports redundant headers contained in the project source code during makefile generation.\nNEBULA_USE_LINKER\nSpecifies the program linker on the system. The available values are:\nbfd, the default value, indicates that ld.bfd is applied as the linker.\nlld, indicates that ld.lld, if installed on the system, is applied as the linker.\ngold, indicates that ld.gold, if installed on the system, is applied as the linker.\nCMAKE_C_COMPILER/CMAKE_CXX_COMPILER\nUsually, CMake locates and uses a C/C++ compiler installed in the host automatically. But if your compiler is not installed at the standard path, or if you want to use a different one, run the command as follows to specify the installation path of the target compiler:\n$ cmake -DCMAKE_C_COMPILER= -DCMAKE_CXX_COMPILER= ..\n$ cmake -DCMAKE_C_COMPILER= -DCMAKE_CXX_COMPILER= ..\nENABLE_CCACHE\nENABLE_CCACHE is ON by default and Ccache (compiler cache) is used to speed up the compiling of NebulaGraph.\nTo disable ccache, setting ENABLE_CCACHE to OFF is not enough. On some platforms, the ccache installation hooks up or precedes the compiler. In such a case, you have to set an environment variable export CCACHE_DISABLE=true or add a line disable=true in ~/.ccache/ccache.conf as well. For more information, see the ccache official documentation.\nNEBULA_THIRDPARTY_ROOT\nNEBULA_THIRDPARTY_ROOT specifies the path where the third party software is installed. By default it is /opt/vesoft/third-party.\nExamine problems\nIf the compiling fails, we suggest you:\nCheck whether the operating system release meets the requirements and whether the memory and hard disk space are sufficient.\nCheck whether the third-party is installed correctly.\nUse make -j1 to reduce the compiling concurrency.","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/2.compile-and-install-nebula-graph/1.install-nebula-graph-by-compiling-the-source-code/","type":"doc"},{"title":"Compile NebulaGraph using Docker","content":"NebulaGraph's source code is written in C++. Compiling NebulaGraph requires certain dependencies which might conflict with host system dependencies, potentially causing compilation failures. Docker offers a solution to this. NebulaGraph provides a Docker image containing the complete compilation environment, ensuring an efficient build process and avoiding host OS conflicts. This guide outlines the steps to compile NebulaGraph using Docker.\nPrerequisites\nBefore you begin:\nDocker: Ensure Docker is installed on your system.\nClone NebulaGraph's Source Code: Clone the repository locally using:\ngit clone --branch release-3.6 https://github.com/vesoft-inc/nebula.git\nThis clones the NebulaGraph source code to a subdirectory named nebula.\nCompilation steps\nPull the NebulaGraph compilation image.\ndocker pull vesoft/nebula-dev:ubuntu2004\nHere, we use the official NebulaGraph compilation image, ubuntu2004. For different versions, see nebula-dev-docker.\nStart the compilation container.\ndocker run -ti \\\n --security-opt seccomp=unconfined \\\n -v \"$PWD\":/home \\\n -w /home \\\n --name nebula_dev \\\n vesoft/nebula-dev:ubuntu2004 \\\n bash\n--security-opt seccomp=unconfined: Disables the seccomp security mechanism to avoid compilation errors.\n-v \"$PWD\":/home: Mounts the local path of the NebulaGraph code to the container's /home directory.\n-w /home: Sets the container's working directory to /home. Any command run inside the container will use this directory as the current directory.\n--name nebula_dev: Assigns a name to the container, making it easier to manage and operate.\nvesoft/nebula-dev:ubuntu2004: Uses the ubuntu2004 version of the vesoft/nebula-dev compilation image.\nbash: Executes the bash command inside the container, entering the container's interactive terminal.\nAfter executing this command, you'll enter an interactive terminal inside the container. To re-enter the container, use docker exec -ti nebula_dev bash.\nCompile NebulaGraph inside the container.\nEnter the NebulaGraph source code directory.\ncd nebula\nCreate a build directory and enter it.\nmkdir build && cd build\nUse CMake to generate the Makefile.\ncmake -DCMAKE_CXX_COMPILER=$TOOLSET_CLANG_DIR/bin/g++ -DCMAKE_C_COMPILER=$TOOLSET_CLANG_DIR/bin/gcc -DENABLE_WERROR=OFF -DCMAKE_BUILD_TYPE=Debug -DENABLE_TESTING=OFF ..\nFor more on CMake, see CMake Parameters.\nCompile NebulaGraph.\n# The -j parameter specifies the number of threads to use.\n# If you have a multi-core CPU, you can use more threads to speed up compilation.\nmake -j2\nCompilation might take some time based on your system performance. \nInstall the Executables and Libraries.\nPost successful compilation, NebulaGraph's binaries and libraries are located in /home/nebula/build. Install them to /usr/local/nebula:\nmake install\nOnce completed, NebulaGraph is compiled and installed in the host directory /usr/local/nebula.\nNext Steps\nStart NebulaGraph Service\nConnect to NebulaGraph","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/2.compile-and-install-nebula-graph/7.compile-using-docker/","type":"doc"},{"title":"Install NebulaGraph with RPM or DEB package","content":"RPM and DEB are common package formats on Linux systems. This topic shows how to quickly install NebulaGraph with the RPM or DEB package.\nPrerequisites\nThe tool wget is installed.\nStep 1: Download the package from cloud service\nDownload the released version.URL:\n//Centos 7\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.el7.x86_64.rpm\n//Centos 8\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.el8.x86_64.rpm\n//Ubuntu 1604\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.ubuntu1604.amd64.deb\n//Ubuntu 1804\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.ubuntu1804.amd64.deb\n//Ubuntu 2004\nhttps://oss-cdn.nebula-graph.io/package//nebula-graph-.ubuntu2004.amd64.deb\nFor example, download the release package 3.6.0 for Centos 7.5:\nwget https://oss-cdn.nebula-graph.io/package/3.6.0/nebula-graph-3.6.0.el7.x86_64.rpm\nwget https://oss-cdn.nebula-graph.io/package/3.6.0/nebula-graph-3.6.0.el7.x86_64.rpm.sha256sum.txt\nDownload the release package 3.6.0 for Ubuntu 1804:\nwget https://oss-cdn.nebula-graph.io/package/3.6.0/nebula-graph-3.6.0.ubuntu1804.amd64.deb\nwget https://oss-cdn.nebula-graph.io/package/3.6.0/nebula-graph-3.6.0.ubuntu1804.amd64.deb.sha256sum.txt\nDownload the nightly version.\nURL:\n//Centos 7\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.el7.x86_64.rpm\n//Centos 8\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.el8.x86_64.rpm\n//Ubuntu 1604\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.ubuntu1604.amd64.deb\n//Ubuntu 1804\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.ubuntu1804.amd64.deb\n//Ubuntu 2004\nhttps://oss-cdn.nebula-graph.io/package/nightly//nebula-graph--nightly.ubuntu2004.amd64.deb\nFor example, download the Centos 7.5 package developed and built in 2021.11.28:\nwget https://oss-cdn.nebula-graph.io/package/nightly/2021.11.28/nebula-graph-2021.11.28-nightly.el7.x86_64.rpm\nwget https://oss-cdn.nebula-graph.io/package/nightly/2021.11.28/nebula-graph-2021.11.28-nightly.el7.x86_64.rpm.sha256sum.txt\nFor example, download the Ubuntu 1804 package developed and built in 2021.11.28:\nwget https://oss-cdn.nebula-graph.io/package/nightly/2021.11.28/nebula-graph-2021.11.28-nightly.ubuntu1804.amd64.deb\nwget https://oss-cdn.nebula-graph.io/package/nightly/2021.11.28/nebula-graph-2021.11.28-nightly.ubuntu1804.amd64.deb.sha256sum.txt\nStep 2: Install NebulaGraph\nUse the following syntax to install with an RPM package.\n$ sudo rpm -ivh --prefix= \nThe option --prefix indicates the installation path. The default path is /usr/local/nebula/.\nFor example, to install an RPM package in the default path for the 3.6.0 version, run the following command.\nsudo rpm -ivh nebula-graph-3.6.0.el7.x86_64.rpm\nUse the following syntax to install with a DEB package.\n$ sudo dpkg -i \nFor example, to install a DEB package for the 3.6.0 version, run the following command.\nsudo dpkg -i nebula-graph-3.6.0.ubuntu1804.amd64.deb\nNext to do\nStart NebulaGraph \nConnect to NebulaGraph","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/2.compile-and-install-nebula-graph/2.install-nebula-graph-by-rpm-or-deb/","type":"doc"},{"title":"Install NebulaGraph graph with the tar.gz file","content":"You can install NebulaGraph by downloading the tar.gz file.\nInstallation steps\nDownload the NebulaGraph tar.gz file using the following address.\nBefore downloading, you need to replace with the version you want to download.\n//Centos 7\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.el7.x86_64.tar.gz\n//Checksum\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.el7.x86_64.tar.gz.sha256sum.txt\n//Centos 8\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.el8.x86_64.tar.gz\n//Checksum\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.el8.x86_64.tar.gz.sha256sum.txt\n//Ubuntu 1604\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.ubuntu1604.amd64.tar.gz\n//Checksum\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.ubuntu1604.amd64.tar.gz.sha256sum.txt\n//Ubuntu 1804\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.ubuntu1804.amd64.tar.gz\n//Checksum\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.ubuntu1804.amd64.tar.gz.sha256sum.txt\n//Ubuntu 2004\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.ubuntu2004.amd64.tar.gz\n//Checksum\nhttps://oss-cdn.nebula-graph.com.cn/package//nebula-graph-.ubuntu2004.amd64.tar.gz.sha256sum.txt\nFor example, to download the NebulaGraph release-3.6 tar.gz file for CentOS 7.5, run the following command:\nwget https://oss-cdn.nebula-graph.com.cn/package/3.6.0/nebula-graph-3.6.0.el7.x86_64.tar.gz\nDecompress the tar.gz file to the NebulaGraph installation directory.\ntar -xvzf -C \ntar.gz_file_name specifies the name of the tar.gz file.\ninstall_path specifies the installation path.\nFor example:\ntar -xvzf nebula-graph-3.6.0.el7.x86_64.tar.gz -C /home/joe/nebula/install\nModify the name of the configuration file.\nEnter the decompressed directory, rename the files nebula-graphd.conf.default, nebula-metad.conf.default, and nebula-storaged.conf.default in the subdirectory etc, and delete .default to apply the default configuration of NebulaGraph.\nSo far, you have installed NebulaGraph successfully.\nNext to do\nManage NebulaGraph services","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/2.compile-and-install-nebula-graph/4.install-nebula-graph-from-tar/","type":"doc"},{"title":"Standalone NebulaGraph","content":"Standalone NebulaGraph merges the Meta, Storage, and Graph services into a single process deployed on a single machine. This topic introduces scenarios, deployment steps, etc. of standalone NebulaGraph.\nBackground\nThe traditional NebulaGraph consists of three services, each service having executable binary files and the corresponding process. Processes communicate with each other by RPC. In standalone NebulaGraph, the three processes corresponding to the three services are combined into one process. For more information about NebulaGraph, see Architecture overview.\nScenarios\nSmall data sizes and low availability requirements. For example, test environments that are limited by the number of machines, scenarios that are only used to verify functionality.\nLimitations\nSingle service instance per machine.\nHigh availability and reliability not supported.\nResource requirements\nFor information about the resource requirements for standalone NebulaGraph, see Software requirements for compiling NebulaGraph.\nSteps\nCurrently, you can only install standalone NebulaGraph with the source code. The steps are similar to those of the multi-process NebulaGraph. You only need to modify the step Generate Makefile with CMake by adding -DENABLE_STANDALONE_VERSION=on to the command. For example:\ncmake -DCMAKE_INSTALL_PREFIX=/usr/local/nebula -DENABLE_TESTING=OFF -DENABLE_STANDALONE_VERSION=on -DCMAKE_BUILD_TYPE=Release .. \nFor more information about installation details, see Install NebulaGraph by compiling the source code.\nAfter installing standalone NebulaGraph, see the topic connect to Service to connect to NebulaGraph databases.\nConfiguration file\nThe path to the configuration file for standalone NebulaGraph is /usr/local/nebula/etc by default.\nYou can run sudo cat nebula-standalone.conf.default to see the file content. The parameters and the corresponding descriptions in the file are generally the same as the configurations for multi-process NebulaGraph except for the following parameters.\nParameter\nPredefined value\nDescription\nmeta_port\n9559\nThe port number of the Meta service.\nstorage_port\n9779\nThe port number of the Storage Service.\nmeta_data_path\ndata/meta\nThe path to Meta data.\nYou can run commands to check configurable parameters and the corresponding descriptions. For details, see Configurations.","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/standalone-deployment/","type":"doc"},{"title":"Deploy a NebulaGraph cluster with RPM/DEB package on multiple servers","content":"For now, NebulaGraph does not provide an official deployment tool. Users can deploy a NebulaGraph cluster with RPM or DEB package manually. This topic provides an example of deploying a NebulaGraph cluster on multiple servers (machines).\nDeployment\nMachine name\nIP address\nNumber of graphd\nNumber of storaged\nNumber of metad\n192.168.10.111\n192.168.10.112\n192.168.10.113\n192.168.10.114\nNone\n192.168.10.115\nNone\nPrerequisites\nPrepare 5 machines for deploying the cluster.\nUse the NTP service to synchronize time in the cluster.\nManual deployment process\nInstall NebulaGraph\nInstall NebulaGraph on each machine in the cluster. Available approaches of installation are as follows.\nInstall NebulaGraph with RPM or DEB package\nInstall NebulaGraph by compiling the source code\nModify the configurations\nTo deploy NebulaGraph according to your requirements, you have to modify the configuration files.\nAll the configuration files for NebulaGraph, including nebula-graphd.conf, nebula-metad.conf, and nebula-storaged.conf, are stored in the etc directory in the installation path. You only need to modify the configuration for the corresponding service on the machines. The configurations that need to be modified for each machine are as follows.\nMachine name\nThe configuration to be modified\nnebula-graphd.conf, nebula-storaged.conf, nebula-metad.conf\nnebula-graphd.conf, nebula-storaged.conf, nebula-metad.conf\nnebula-graphd.conf, nebula-storaged.conf, nebula-metad.conf\nnebula-graphd.conf, nebula-storaged.conf\nnebula-graphd.conf, nebula-storaged.conf\nUsers can refer to the content of the following configurations, which only show part of the cluster settings. The hidden content uses the default setting so that users can better understand the relationship between the servers in the NebulaGraph cluster.\nDeploy machine A\nnebula-graphd.conf\n########## networking ##########\n# Comma separated Meta Server Addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-graphd process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.111\n# Network device to listen on\n--listen_netdev=any\n# Port to listen on\n--port=9669\nnebula-storaged.conf\n########## networking ##########\n# Comma separated Meta server addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-storaged process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.111\n# Storage daemon listening port\n--port=9779\nnebula-metad.conf\n########## networking ##########\n# Comma separated Meta Server addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-metad process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.111\n# Meta daemon listening port\n--port=9559\nDeploy machine B\nnebula-graphd.conf\n########## networking ##########\n# Comma separated Meta Server Addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-graphd process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.112\n# Network device to listen on\n--listen_netdev=any\n# Port to listen on\n--port=9669\nnebula-storaged.conf\n########## networking ##########\n# Comma separated Meta server addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-storaged process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.112\n# Storage daemon listening port\n--port=9779\nnebula-metad.conf\n########## networking ##########\n# Comma separated Meta Server addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-metad process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.112\n# Meta daemon listening port\n--port=9559\nDeploy machine C\nnebula-graphd.conf\n########## networking ##########\n# Comma separated Meta Server Addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-graphd process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.113\n# Network device to listen on\n--listen_netdev=any\n# Port to listen on\n--port=9669\nnebula-storaged.conf\n########## networking ##########\n# Comma separated Meta server addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-storaged process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.113\n# Storage daemon listening port\n--port=9779\nnebula-metad.conf\n########## networking ##########\n# Comma separated Meta Server addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-metad process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.113\n# Meta daemon listening port\n--port=9559\nDeploy machine D\nnebula-graphd.conf\n########## networking ##########\n# Comma separated Meta Server Addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-graphd process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.114\n# Network device to listen on\n--listen_netdev=any\n# Port to listen on\n--port=9669\nnebula-storaged.conf\n########## networking ##########\n# Comma separated Meta server addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-storaged process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.114\n# Storage daemon listening port\n--port=9779\nDeploy machine E\nnebula-graphd.conf\n########## networking ##########\n# Comma separated Meta Server Addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-graphd process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.115\n# Network device to listen on\n--listen_netdev=any\n# Port to listen on\n--port=9669\nnebula-storaged.conf\n########## networking ##########\n# Comma separated Meta server addresses\n--meta_server_addrs=192.168.10.111:9559,192.168.10.112:9559,192.168.10.113:9559\n# Local IP used to identify the nebula-storaged process.\n# Change it to an address other than loopback if the service is distributed or\n# will be accessed remotely.\n--local_ip=192.168.10.115\n# Storage daemon listening port\n--port=9779\nStart the cluster\nStart the corresponding service on each machine. Descriptions are as follows.\nMachine name\nThe process to be started\ngraphd, storaged, metad\ngraphd, storaged, metad\ngraphd, storaged, metad\ngraphd, storaged\ngraphd, storaged\nThe command to start the NebulaGraph services is as follows.\nsudo /usr/local/nebula/scripts/nebula.service start \nCheck the cluster status\nInstall the native CLI client NebulaGraph Console, then connect to any machine that has started the graphd process, run ADD HOSTS command to add storage hosts, and run SHOW HOSTS to check the cluster status. For example:\n$ ./nebula-console --addr 192.168.10.111 --port 9669 -u root -p nebula\n2021/05/25 01:41:19 [INFO] connection pool is initialized successfully\nWelcome to NebulaGraph!\n> ADD HOSTS 192.168.10.111:9779, 192.168.10.112:9779, 192.168.10.113:9779, 192.168.10.114:9779, 192.168.10.115:9779;\n> SHOW HOSTS;","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/2.compile-and-install-nebula-graph/deploy-nebula-graph-cluster/","type":"doc"},{"title":"Deploy NebulaGraph with Docker Compose","content":"Using Docker Compose can quickly deploy NebulaGraph services based on the prepared configuration file. It is only recommended to use this method when testing functions of NebulaGraph.\nPrerequisites\nYou have installed the following applications on your host.\nApplication\nRecommended version\nOfficial installation reference\nDocker\nLatest\nInstall Docker Engine\nDocker Compose\nLatest\nInstall Docker Compose\nGit\nLatest\nDownload Git\nIf you are deploying NebulaGraph as a non-root user, grant the user with Docker-related privileges. For detailed instructions, see Manage Docker as a non-root user.\nYou have started the Docker service on your host.\nIf you have already deployed another version of NebulaGraph with Docker Compose on your host, to avoid compatibility issues, you need to delete the nebula-docker-compose/data directory.\nDeploy NebulaGraph\nClone the 3.6.0 branch of the nebula-docker-compose repository to your host with Git.\n$ git clone -b release-3.6 https://github.com/vesoft-inc/nebula-docker-compose.git\nGo to the nebula-docker-compose directory.\n$ cd nebula-docker-compose/\nRun the following command to start all the NebulaGraph services.\n[nebula-docker-compose]$ docker-compose up -d\nCreating nebula-docker-compose_metad0_1 ... done\nCreating nebula-docker-compose_metad2_1 ... done\nCreating nebula-docker-compose_metad1_1 ... done\nCreating nebula-docker-compose_graphd2_1 ... done\nCreating nebula-docker-compose_graphd_1 ... done\nCreating nebula-docker-compose_graphd1_1 ... done\nCreating nebula-docker-compose_storaged0_1 ... done\nCreating nebula-docker-compose_storaged2_1 ... done\nCreating nebula-docker-compose_storaged1_1 ... done\nConnect to NebulaGraph\nThere are two ways to connect to NebulaGraph:\nConnected with Nebula Console outside the container. Because the external mapping port for the Graph service is also fixed as 9669 in the container's configuration file, you can connect directly through the default port. For details, see Connect to NebulaGraph.\nLog into the container installed NebulaGraph Console, then connect to the Graph service. This section describes this approach.\nRun the following command to view the name of NebulaGraph Console docker container.\n$ docker-compose ps\n Name Command State Ports\n--------------------------------------------------------------------------------------------\nnebula-docker-compose_console_1 sh -c sleep 3 && Up\n nebula-co ...\n......\nRun the following command to enter the NebulaGraph Console docker container.\ndocker exec -it nebula-docker-compose_console_1 /bin/sh\n/ #\nConnect to NebulaGraph with NebulaGraph Console.\n/ # ./usr/local/bin/nebula-console -u -p --address=graphd --port=9669\nRun the following commands to view the cluster state.\n SHOW HOSTS;\nRun exit twice to switch back to your terminal (shell).\nCheck the NebulaGraph service status and ports\nRun docker-compose ps to list all the services of NebulaGraph and their status and ports.\n$ docker-compose ps\nnebula-docker-compose_console_1 sh -c sleep 3 && Up\n nebula-co ...\nnebula-docker-compose_graphd1_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49174->19669/tcp,:::49174->19669/tcp, 0.0.0.0:49171->19670/tcp,:::49171->19670/tcp, 0.0.0.0:49177->9669/tcp,:::49177->9669/tcp\nnebula-docker-compose_graphd2_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49175->19669/tcp,:::49175->19669/tcp, 0.0.0.0:49172->19670/tcp,:::49172->19670/tcp, 0.0.0.0:49178->9669/tcp,:::49178->9669/tcp\nnebula-docker-compose_graphd_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49180->19669/tcp,:::49180->19669/tcp, 0.0.0.0:49179->19670/tcp,:::49179->19670/tcp, 0.0.0.0:9669->9669/tcp,:::9669->9669/tcp\nnebula-docker-compose_metad0_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49157->19559/tcp,:::49157->19559/tcp, 0.0.0.0:49154->19560/tcp,:::49154->19560/tcp, 0.0.0.0:49160->9559/tcp,:::49160->9559/tcp, 9560/tcp\nnebula-docker-compose_metad1_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49156->19559/tcp,:::49156->19559/tcp, 0.0.0.0:49153->19560/tcp,:::49153->19560/tcp, 0.0.0.0:49159->9559/tcp,:::49159->9559/tcp, 9560/tcp\nnebula-docker-compose_metad2_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49158->19559/tcp,:::49158->19559/tcp, 0.0.0.0:49155->19560/tcp,:::49155->19560/tcp, 0.0.0.0:49161->9559/tcp,:::49161->9559/tcp, 9560/tcp\nnebula-docker-compose_storaged0_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49166->19779/tcp,:::49166->19779/tcp, 0.0.0.0:49163->19780/tcp,:::49163->19780/tcp, 9777/tcp, 9778/tcp, 0.0.0.0:49169->9779/tcp,:::49169->9779/tcp, 9780/tcp\nnebula-docker-compose_storaged1_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49165->19779/tcp,:::49165->19779/tcp, 0.0.0.0:49162->19780/tcp,:::49162->19780/tcp, 9777/tcp, 9778/tcp, 0.0.0.0:49168->9779/tcp,:::49168->9779/tcp, 9780/tcp\nnebula-docker-compose_storaged2_1 /usr/local/nebula/bin/nebu ... Up 0.0.0.0:49167->19779/tcp,:::49167->19779/tcp, 0.0.0.0:49164->19780/tcp,:::49164->19780/tcp, 9777/tcp, 9778/tcp, 0.0.0.0:49170->9779/tcp,:::49170->9779/tcp, 9780/tcp\nIf the service is abnormal, you can first confirm the abnormal container name (such as nebula-docker-compose_graphd2_1).\nThen you can execute docker ps to view the corresponding CONTAINER ID (such as 2a6c56c405f5).\n[nebula-docker-compose]$ docker ps\nCONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\n2a6c56c405f5 vesoft/nebula-graphd:nightly \"/usr/local/nebula/b…\" 36 minutes ago Up 36 minutes (healthy) 0.0.0.0:49230->9669/tcp, 0.0.0.0:49229->19669/tcp, 0.0.0.0:49228->19670/tcp nebula-docker-compose_graphd2_1\n7042e0a8e83d vesoft/nebula-storaged:nightly \"./bin/nebula-storag…\" 36 minutes ago Up 36 minutes (healthy) 9777-9778/tcp, 9780/tcp, 0.0.0.0:49227->9779/tcp, 0.0.0.0:49226->19779/tcp, 0.0.0.0:49225->19780/tcp nebula-docker-compose_storaged2_1\n18e3ea63ad65 vesoft/nebula-storaged:nightly \"./bin/nebula-storag…\" 36 minutes ago Up 36 minutes (healthy) 9777-9778/tcp, 9780/tcp, 0.0.0.0:49219->9779/tcp, 0.0.0.0:49218->19779/tcp, 0.0.0.0:49217->19780/tcp nebula-docker-compose_storaged0_1\n4dcabfe8677a vesoft/nebula-graphd:nightly \"/usr/local/nebula/b…\" 36 minutes ago Up 36 minutes (healthy) 0.0.0.0:49224->9669/tcp, 0.0.0.0:49223->19669/tcp, 0.0.0.0:49222->19670/tcp nebula-docker-compose_graphd1_1\na74054c6ae25 vesoft/nebula-graphd:nightly \"/usr/local/nebula/b…\" 36 minutes ago Up 36 minutes (healthy) 0.0.0.0:9669->9669/tcp, 0.0.0.0:49221->19669/tcp, 0.0.0.0:49220->19670/tcp nebula-docker-compose_graphd_1\n880025a3858c vesoft/nebula-storaged:nightly \"./bin/nebula-storag…\" 36 minutes ago Up 36 minutes (healthy) 9777-9778/tcp, 9780/tcp, 0.0.0.0:49216->9779/tcp, 0.0.0.0:49215->19779/tcp, 0.0.0.0:49214->19780/tcp nebula-docker-compose_storaged1_1\n45736a32a23a vesoft/nebula-metad:nightly \"./bin/nebula-metad …\" 36 minutes ago Up 36 minutes (healthy) 9560/tcp, 0.0.0.0:49213->9559/tcp, 0.0.0.0:49212->19559/tcp, 0.0.0.0:49211->19560/tcp nebula-docker-compose_metad0_1\n3b2c90eb073e vesoft/nebula-metad:nightly \"./bin/nebula-metad …\" 36 minutes ago Up 36 minutes (healthy) 9560/tcp, 0.0.0.0:49207->9559/tcp, 0.0.0.0:49206->19559/tcp, 0.0.0.0:49205->19560/tcp nebula-docker-compose_metad2_1\n7bb31b7a5b3f vesoft/nebula-metad:nightly \"./bin/nebula-metad …\" 36 minutes ago Up 36 minutes (healthy) 9560/tcp, 0.0.0.0:49210->9559/tcp, 0.0.0.0:49209->19559/tcp, 0.0.0.0:49208->19560/tcp nebula-docker-compose_metad1_1\nUse the CONTAINER ID to log in the container and troubleshoot.\nnebula-docker-compose]$ docker exec -it 2a6c56c405f5 bash\n[root@2a6c56c405f5 nebula]#\nCheck the service data and logs\nAll the data and logs of NebulaGraph are stored persistently in the nebula-docker-compose/data and nebula-docker-compose/logs directories.\nThe structure of the directories is as follows:\nnebula-docker-compose/\n |-- docker-compose.yaml\n ├── data\n │   ├── meta0\n │   ├── meta1\n │   ├── meta2\n │   ├── storage0\n │   ├── storage1\n │   └── storage2\n └── logs\n ├── graph\n ├── graph1\n ├── graph2\n ├── meta0\n ├── meta1\n ├── meta2\n ├── storage0\n ├── storage1\n └── storage2\nStop the NebulaGraph services\nYou can run the following command to stop the NebulaGraph services:\n$ docker-compose down\nThe following information indicates you have successfully stopped the NebulaGraph services:\nStopping nebula-docker-compose_console_1 ... done\nStopping nebula-docker-compose_graphd1_1 ... done\nStopping nebula-docker-compose_graphd_1 ... done\nStopping nebula-docker-compose_graphd2_1 ... done\nStopping nebula-docker-compose_storaged1_1 ... done\nStopping nebula-docker-compose_storaged0_1 ... done\nStopping nebula-docker-compose_storaged2_1 ... done\nStopping nebula-docker-compose_metad2_1 ... done\nStopping nebula-docker-compose_metad0_1 ... done\nStopping nebula-docker-compose_metad1_1 ... done\nRemoving nebula-docker-compose_console_1 ... done\nRemoving nebula-docker-compose_graphd1_1 ... done\nRemoving nebula-docker-compose_graphd_1 ... done\nRemoving nebula-docker-compose_graphd2_1 ... done\nRemoving nebula-docker-compose_storaged1_1 ... done\nRemoving nebula-docker-compose_storaged0_1 ... done\nRemoving nebula-docker-compose_storaged2_1 ... done\nRemoving nebula-docker-compose_metad2_1 ... done\nRemoving nebula-docker-compose_metad0_1 ... done\nRemoving nebula-docker-compose_metad1_1 ... done\nRemoving network nebula-docker-compose_nebula-net\nModify configurations\nThe configuration file of NebulaGraph deployed by Docker Compose is nebula-docker-compose/docker-compose.yaml. To make the new configuration take effect, modify the configuration in this file and restart the service.\nFor more instructions, see Configurations.\nFAQ\nHow to fix the docker mapping to external ports?\nTo set the ports of corresponding services as fixed mapping, modify the docker-compose.yaml in the nebula-docker-compose directory. For example:\ngraphd:\n image: vesoft/nebula-graphd:release-3.6\n ...\n ports:\n - 9669:9669\n - 19669\n - 19670\n9669:9669 indicates the internal port 9669 is uniformly mapped to external ports, while 19669 indicates the internal port 19669 is randomly mapped to external ports.\nHow to upgrade or update the docker images of NebulaGraph services\nIn the nebula-docker-compose/docker-compose.yaml file, change all the image values to the required image version.\nIn the nebula-docker-compose directory, run docker-compose pull to update the images of the Graph Service, Storage Service, Meta Service, and NebulaGraph Console.\nRun docker-compose up -d to start the NebulaGraph services again.\nAfter connecting to NebulaGraph with NebulaGraph Console, run SHOW HOSTS GRAPH, SHOW HOSTS STORAGE, or SHOW HOSTS META to check the version of the responding service respectively.\nERROR: toomanyrequests when docker-compose pull\nYou may meet the following error.\nERROR: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit.\nYou have met the rate limit of Docker Hub. Learn more on Understanding Docker Hub Rate Limiting.\nHow to update the NebulaGraph Console client\nThe command docker-compose pull updates both the NebulaGraph services and the NebulaGraph Console.\nRelated documents\nInstall and deploy NebulaGraph with the source code\nInstall NebulaGraph by RPM or DEB\nConnect to NebulaGraph","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/2.compile-and-install-nebula-graph/3.deploy-nebula-graph-with-docker-compose/","type":"doc"},{"title":"Install NebulaGraph with ecosystem tools","content":"You can install the NebulaGraph Community Edition with the following ecosystem tools:\nNebulaGraph Operator\nInstallation details\nTo install NebulaGraph with NebulaGraph Operator, see Install NebulaGraph clusters.","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/2.compile-and-install-nebula-graph/6.deploy-nebula-graph-with-peripherals/","type":"doc"},{"title":"Manage NebulaGraph Service","content":"NebulaGraph supports managing services with scripts. \nManage services with script\nYou can use the nebula.service script to start, stop, restart, terminate, and check the NebulaGraph services.\nSyntax\n$ sudo /usr/local/nebula/scripts/nebula.service\n[-v] [-c ]\n\n\nParameter\nDescription\n-v\nDisplay detailed debugging information.\n-c\nSpecify the configuration file path. The default path is /usr/local/nebula/etc/.\nstart\nStart the target services.\nstop\nStop the target services.\nrestart\nRestart the target services.\nkill\nTerminate the target services.\nstatus\nCheck the status of the target services.\nmetad\nSet the Meta Service as the target service.\ngraphd\nSet the Graph Service as the target service.\nstoraged\nSet the Storage Service as the target service.\nall\nSet all the NebulaGraph services as the target services.\nStart NebulaGraph\nRun the following command to start NebulaGraph.\n$ sudo /usr/local/nebula/scripts/nebula.service start all\n[INFO] Starting nebula-metad...\n[INFO] Done\n[INFO] Starting nebula-graphd...\n[INFO] Done\n[INFO] Starting nebula-storaged...\n[INFO] Done\nStop NebulaGraph\nRun the following command to stop NebulaGraph.\n$ sudo /usr/local/nebula/scripts/nebula.service stop all\n[INFO] Stopping nebula-metad...\n[INFO] Done\n[INFO] Stopping nebula-graphd...\n[INFO] Done\n[INFO] Stopping nebula-storaged...\n[INFO] Done\nCheck the service status\nRun the following command to check the service status of NebulaGraph.\n$ sudo /usr/local/nebula/scripts/nebula.service status all\nNebulaGraph is running normally if the following information is returned.\nINFO] nebula-metad(33fd35e): Running as 29020, Listening on 9559\n[INFO] nebula-graphd(33fd35e): Running as 29095, Listening on 9669\n[WARN] nebula-storaged after v3.0.0 will not start service until it is added to cluster.\n[WARN] See Manage Storage hosts:ADD HOSTS in https://docs.nebula-graph.io/\n[INFO] nebula-storaged(33fd35e): Running as 29147, Listening on 9779\nIf the returned result is similar to the following one, there is a problem. You may also go to the NebulaGraph community for help.[INFO] nebula-metad: Running as 25600, Listening on 9559\n[INFO] nebula-graphd: Exited\n[INFO] nebula-storaged: Running as 25646, Listening on 9779\nThe NebulaGraph services consist of the Meta Service, Graph Service, and Storage Service. The configuration files for all three services are stored in the /usr/local/nebula/etc/ directory by default. You can check the configuration files according to the returned result to troubleshoot problems.\nNext to do\nConnect to NebulaGraph","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/manage-service/","type":"doc"},{"title":"Connect to NebulaGraph","content":"This topic provides basic instruction on how to use the native CLI client NebulaGraph Console to connect to NebulaGraph.\nNebulaGraph supports multiple types of clients, including a CLI client, a GUI client, and clients developed in popular programming languages. For more information, see the client list.\nPrerequisites\nYou have started NebulaGraph services.\nThe machine on which you plan to run NebulaGraph Console has network access to the Graph Service of NebulaGraph.\nThe NebulaGraph Console version is compatible with the NebulaGraph version.\nSteps\nOn the NebulaGraph Console releases page, select a NebulaGraph Console version and click Assets.\nIn the Assets area, find the correct binary file for the machine where you want to run NebulaGraph Console and download the file to the machine.\n(Optional) Rename the binary file to nebula-console for convenience.\nOn the machine to run NebulaGraph Console, grant the execute permission of the nebula-console binary file to the user.\n$ chmod 111 nebula-console\nIn the command line interface, change the working directory to the one where the nebula-console binary file is stored.\nRun the following command to connect to NebulaGraph.\nFor Linux or macOS:\n$ ./nebula-console -addr -port -u -p \n[-t 120] [-e \"nGQL_statement\" | -f filename.nGQL]\nFor Windows:\n> nebula-console.exe -addr -port -u -p \n[-t 120] [-e \"nGQL_statement\" | -f filename.nGQL]\nParameter descriptions are as follows:\nParameter\nDescription\n-h/-help\nShows the help menu.\n-addr/-address\nSets the IP (or hostname) of the Graph service. The default address is 127.0.0.1. \n-P/-port\nSets the port number of the graphd service. The default port number is 9669.\n-u/-user\nSets the username of your NebulaGraph account. Before enabling authentication, you can use any existing username. The default username is root.\n-p/-password\nSets the password of your NebulaGraph account. Before enabling authentication, you can use any characters as the password.\n-t/-timeout\nSets an integer-type timeout threshold of the connection. The unit is millisecond. The default value is 120.\n-e/-eval\nSets a string-type nGQL statement. The nGQL statement is executed once the connection succeeds. The connection stops after the result is returned.\n-f/-file\nSets the path of an nGQL file. The nGQL statements in the file are executed once the connection succeeds. The result will be returned and the connection stops then.\n-enable_ssl\nEnables SSL encryption when connecting to NebulaGraph.\n-ssl_root_ca_path\nSets the storage path of the certification authority file.\n-ssl_cert_path\nSets the storage path of the certificate file.\n-ssl_private_key_path\nSets the storage path of the private key file.\nFor information on more parameters, see the project repository.","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/connect-to-nebula-graph/","type":"doc"},{"title":"Manage Storage hosts","content":"Starting from NebulaGraph 3.0.0, setting Storage hosts in the configuration files only registers the hosts on the Meta side, but does not add them into the cluster. You must run the ADD HOSTS statement to add the Storage hosts.\nPrerequisites\nYou have connected to the NebulaGraph database.\nAdd Storage hosts\nAdd the Storage hosts to a NebulaGraph cluster.\n ADD HOSTS : [,: ...];\n ADD HOSTS \"\": [,\"\": ...];\nDrop Storage hosts\nDelete the Storage hosts from cluster.\n DROP HOSTS : [,: ...];\n DROP HOSTS \"\": [,\"\": ...];\nView Storage hosts\nView the Storage hosts in the cluster.\n SHOW HOSTS STORAGE;","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/manage-storage-host/","type":"doc"},{"title":"Upgrade NebulaGraph to 3.6.0","content":"This topic describes how to upgrade NebulaGraph from version 2.x and 3.x to 3.6.0, taking upgrading from version 2.6.1 to 3.6.0 as an example.\nApplicable source versions\nThis topic applies to upgrading NebulaGraph from 2.5.0 and later 2.x, and 3.x versions to 3.6.0. It does not apply to historical versions earlier than 2.5.0, including the 1.x versions.\nTo upgrade NebulaGraph from historical versions to 3.6.0:\nUpgrade it to the latest 2.5 version according to the docs of that version.\nFollow this topic to upgrade it to 3.6.0.\nLimitations\nRolling Upgrade is not supported. You must stop all the NebulaGraph services before the upgrade.\nThere is no upgrade script. You have to manually upgrade each server in the cluster.\nThis topic does not apply to scenarios where NebulaGraph is deployed with Docker, including Docker Swarm, Docker Compose, and K8s.\nYou must upgrade the old NebulaGraph services on the same machines they are deployed. DO NOT change the IP addresses, configuration files of the machines, and DO NOT change the cluster topology.\nKnown issues that could cause data loss are listed on GitHub known issues. The issues are all related to altering schema or default values.\nDO NOT use soft links to switch the data directories.\nYou must have the sudo privileges to complete the steps in this topic.\nUpgrade influences\nClient compatibility\nAfter the upgrade, you will not be able to connect to NebulaGraph from old clients. You will need to upgrade all clients to a version compatible with NebulaGraph 3.6.0.\nConfiguration changes\nA few configuration parameters have been changed. For more information, see the release notes and configuration docs.\nnGQL compatibility\nThe nGQL syntax is partially incompatible:\nDisable the YIELD clause to return custom variables.\nThe YIELD clause is required in the FETCH, GO, LOOKUP, FIND PATH and GET SUBGRAPH statements.\nIt is required to specify a tag to query properties of a vertex in a MATCH statement. For example, from return v.name to return v.player.name.\nFull-text indexes\nBefore upgrading a NebulaGraph cluster with full-text indexes deployed, you must manually delete the full-text indexes in Elasticsearch, and then run the SIGN IN command to log into ES and recreate the indexes after the upgrade is complete. To manually delete the full-text indexes in Elasticsearch, you can use the curl command curl -XDELETE -u : ':/', for example, curl -XDELETE -u elastic:elastic 'http://192.168.8.xxx:9200/nebula_index_2534'. If no username and password are set for Elasticsearch, you can omit the -u : part.\nPreparations before the upgrade\nDownload the package of NebulaGraph 3.6.0 according to your operating system and system architecture. You need the binary files during the upgrade. Find the package on the download page.\nLocate the data files based on the value of the data_path parameters in the Storage and Meta configurations, and backup the data files. The default paths are nebula/data/storage and nebula/data/meta.\nBackup the configuration files.\nCollect the statistics of all graph spaces before the upgrade. After the upgrade, you can collect again and compare the results to make sure that no data is lost. To collect the statistics:\nRun SUBMIT JOB STATS.\nRun SHOW JOBS and record the result.\nUpgrade steps\nStop all NebulaGraph services.\n/scripts/nebula.service stop all\nnebula_install_path indicates the installation path of NebulaGraph.\nThe storaged progress needs around 1 minute to flush data. You can run nebula.service status all to check if all services are stopped. For more information about starting and stopping services, see Manage services.\nIn the target path where you unpacked the package, use the binaries in the bin directory to replace the old binaries in the bin directory in the NebulaGraph installation path.\nModify the following parameters in all Graph configuration files to accommodate the value range of the new version. If the parameter values are within the specified range, skip this step.\nSet a value in [1,604800] for session_idle_timeout_secs. The recommended value is 28800.\nSet a value in [1,604800] for client_idle_timeout_secs. The recommended value is 28800.\nThe default values of these parameters in the 2.x versions are not within the range of the new version. If you do not change the default values, the upgrade will fail. For detailed parameter description, see Graph Service Configuration.\nStart all Meta services.\n/scripts/nebula-metad.service start\nOnce started, the Meta services take several seconds to elect a leader.\nTo verify that Meta services are all started, you can start any Graph server, connect to it through NebulaGraph Console, and run SHOW HOSTS meta and SHOW META LEADER. If the status of Meta services are correctly returned, the services are successfully started.\nStart all the Graph and Storage services.\nConnect to the new version of NebulaGraph to verify that services are available and data are complete. For how to connect, see Connect to NebulaGraph.\nCurrently, there is no official way to check whether the upgrade is successful. You can run the following reference statements to test the upgrade:\n SHOW HOSTS;\n SHOW HOSTS storage;\n SHOW SPACES;\n USE \n SHOW PARTS;\n SUBMIT JOB STATS;\n SHOW STATS;\n MATCH (v) RETURN v LIMIT 5;\nYou can also test against new features in version 3.6.0.\nUpgrade failure and rollback\nIf the upgrade fails, stop all NebulaGraph services of the new version, recover the old configuration files and binaries, and start the services of the old version.\nAll NebulaGraph clients in use must be switched to the old version.\nFAQ\nCan I write through the client during the upgrade?\nA: No. You must stop all NebulaGraph services during the upgrade.\nThe Space 0 not found warning message during the upgrade process\nWhen the Space 0 not found warning message appears during the upgrade process, you can ignore it. The space 0 is used to store meta information about the Storage service and does not contain user data, so it will not affect the upgrade.\nHow to upgrade if a machine has only the Graph Service, but not the Storage Service?\nA: You only need to update the configuration files and binaries of the Graph Service.\nHow to resolve the error Permission denied?\nA: Try again with the sudo privileges.\nIs there any change in gflags?\nA: Yes. For more information, see the release notes and configuration docs.\nIs there a tool or solution for verifying data consistency after the upgrade?\nA: No. But if you only want to check the number of vertices and edges, run SUBMIT JOB STATS and SHOW STATS after the upgrade, and compare the result with the result that you recorded before the upgrade.\nHow to solve the issue that Storage is OFFLINE and Leader count is 0?\nA: Run the following statement to add the Storage hosts into the cluster manually.\nADD HOSTS :[, : ...];\nFor example:\nADD HOSTS 192.168.10.100:9779, 192.168.10.101:9779, 192.168.10.102:9779;\nIf the issue persists, ask for help on the forum or GitHub.\nWhy the job type changed after the upgrade, but job ID remains the same?\nA: SHOW JOBS depends on an internal ID to identify job types, but in NebulaGraph 2.5.0 the internal ID changed in this pull request, so this issue happens after upgrading from a version earlier than 2.5.0.","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/3.upgrade-nebula-graph/upgrade-nebula-comm/","type":"doc"},{"title":"Uninstall NebulaGraph","content":"This topic describes how to uninstall NebulaGraph.\nPrerequisite\nThe NebulaGraph services should be stopped before the uninstallation. For more information, see Manage NebulaGraph services.\nStep 1: Delete data files of the Storage and Meta Services\nIf you have modified the data_path in the configuration files for the Meta Service and Storage Service, the directories where NebulaGraph stores data may not be in the installation path of NebulaGraph. Check the configuration files to confirm the data paths, and then manually delete the directories to clear all data.\nCheck the Storage Service disk settings. For example:\n########## Disk ##########\n# Root data path. Split by comma. e.g. --data_path=/disk1/path1/,/disk2/path2/\n# One path per Rocksdb instance.\n--data_path=/nebula/data/storage\nCheck the Metad Service configurations and find the corresponding metadata directories.\nDelete the data and the directories found in step 2.\nStep 2: Delete the installation directories\nThe default installation path is /usr/local/nebula, which is specified by --prefix while installing NebulaGraph.\nUninstall NebulaGraph deployed with source code\nFind the installation directories of NebulaGraph, and delete them all.\nUninstall NebulaGraph deployed with RPM packages\nRun the following command to get the NebulaGraph version.\n$ rpm -qa | grep \"nebula\"\nThe return message is as follows.\nnebula-graph-3.6.0-1.x86_64\nRun the following command to uninstall NebulaGraph.\nsudo rpm -e \nFor example:\nsudo rpm -e nebula-graph-3.6.0-1.x86_64\nDelete the installation directories.\nUninstall NebulaGraph deployed with DEB packages\nRun the following command to get the NebulaGraph version.\n$ dpkg -l | grep \"nebula\"\nThe return message is as follows.\nii nebula-graph 3.6.0 amd64 NebulaGraph Package built using CMake\nRun the following command to uninstall NebulaGraph.\nsudo dpkg -r \nFor example:\nsudo dpkg -r nebula-graph\nDelete the installation directories.\nUninstall NebulaGraph deployed with Docker Compose\nIn the nebula-docker-compose directory, run the following command to stop the NebulaGraph services.\ndocker-compose down -v\nDelete the nebula-docker-compose directory.","url":"https://docs.nebula-graph.io/3.6.0/4.deployment-and-installation/4.uninstall-nebula-graph/","type":"doc"},{"title":"Configurations","content":"NebulaGraph builds the configurations based on the gflags repository. Most configurations are flags. When the NebulaGraph service starts, it will get the configuration information from Configuration files by default. Configurations that are not in the file apply the default values.\nGet the configuration list and descriptions\nUse the following command to get all the configuration information of the service corresponding to the binary file:\n --help\nFor example:\n# Get the help information from Meta\n$ /usr/local/nebula/bin/nebula-metad --help\n# Get the help information from Graph\n$ /usr/local/nebula/bin/nebula-graphd --help\n# Get the help information from Storage\n$ /usr/local/nebula/bin/nebula-storaged --help\nThe above examples use the default storage path /usr/local/nebula/bin/. If you modify the installation path of NebulaGraph, use the actual path to query the configurations.\nGet configurations\nUse the curl command to get the value of the running configurations.\nFor example:\n# Get the running configurations from Meta\ncurl 127.0.0.1:19559/flags\n# Get the running configurations from Graph\ncurl 127.0.0.1:19669/flags\n# Get the running configurations from Storage\ncurl 127.0.0.1:19779/flags\nUtilizing the -s or `-silent option allows for the concealment of the progress bar and error messages. For example:\ncurl -s 127.0.0.1:19559/flags\nConfiguration files\nConfiguration files for clusters installed from source, with an RPM/DEB package, or a TAR package\nNebulaGraph provides two initial configuration files for each service, .conf.default and .conf.production. You can use them in different scenarios conveniently. For clusters installed from source and with a RPM/DEB package, the default path is /usr/local/nebula/etc/. For clusters installed with a TAR package, the path is //etc.\nThe configuration values in the initial configuration file are for reference only and can be adjusted according to actual needs. To use the initial configuration file, choose one of the above two files and delete the suffix .default or .production to make it valid.\nThe initial configuration files corresponding to each service are as follows.\nNebulaGraph service\nInitial configuration file\nDescription\nMeta\nnebula-metad.conf.default and nebula-metad.conf.production\nMeta service configuration\nGraph\nnebula-graphd.conf.default and nebula-graphd.conf.production\nGraph service configuration\nStorage\nnebula-storaged.conf.default and nebula-storaged.conf.production\nStorage service configuration\nEach initial configuration file of all services contains local_config. The default value is true, which means that the NebulaGraph service will get configurations from its configuration files and start it.\nConfiguration files for clusters installed with Docker Compose\nFor clusters installed with Docker Compose, the configuration file's default installation path of the cluster is /nebula-docker-compose/docker-compose.yaml. The parameters in the command field of the file are the launch parameters for each service. \nConfiguration files for clusters installed with NebulaGraph Operator\nFor clusters installed with Kubectl through NebulaGraph Operator, the configuration file's path is the path of the cluster YAML file. You can modify the configuration of each service through the spec.{graphd|storaged|metad}.config parameter. \nModify configurations\nYou can modify the configurations of NebulaGraph in the configuration file or use commands to dynamically modify configurations.\nModifying configurations in the configuration file\nBy default, each NebulaGraph service gets configured from its configuration files. You can modify configurations and make them valid according to the following steps:\nFor clusters installed from source, with a RPM/DEB, or a TAR package\nUse a text editor to modify the configuration files of the target service and save the modification.\nChoose an appropriate time to restart all NebulaGraph services to make the modifications valid.\nFor clusters installed with Docker Compose\nIn the /nebula-docker-compose/docker-compose.yaml file, modify the configurations of the target service.\nIn the nebula-docker-compose directory, run the command docker-compose up -d to restart the service involving configuration modifications.\nFor clusters installed with Kubectl\nFor details, see Customize configuration parameters for a NebulaGraph cluster.\nDynamically modifying configurations using command\nYou can dynamically modify the configuration of NebulaGraph by using the curl command. For example, to modify the wal_ttl parameter of the Storage service to 600, use the following command:\ncurl -X PUT -H \"Content-Type: application/json\" -d'{\"wal_ttl\":\"600\"}' -s \"http://192.168.15.6:19779/flags\"\nIn this command, {\"wal_ttl\":\"600\"} specifies the configuration parameter and its value to be modified, and 192.168.15.6:19779 specifies the IP address and HTTP port number of the Storage service.","url":"https://docs.nebula-graph.io/3.6.0/5.configurations-and-logs/1.configurations/1.configurations/","type":"doc"},{"title":"Meta Service configuration","content":"NebulaGraph provides two initial configuration files for the Meta Service, nebula-metad.conf.default and nebula-metad.conf.production. Users can use them in different scenarios conveniently. The default file path is /usr/local/nebula/etc/.\nHow to use the configuration files\nTo use the initial configuration file, choose one of the above two files and delete the suffix .default or .production from the initial configuration file for the Meta Service to apply the configurations defined in it.\nAbout parameter values\nIf a parameter is not set in the configuration file, NebulaGraph uses the default value. Not all parameters are predefined. And the predefined parameters in the two initial configuration files are different. This topic uses the parameters in nebula-metad.conf.default.\nFor all parameters and their current values, see Configurations.\nBasics configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\ndaemonize\ntrue\nWhen set to true, the process is a daemon process.\nNo\npid_file\npids/nebula-metad.pid\nThe file that records the process ID.\nNo\ntimezone_name\nSpecifies the NebulaGraph time zone. This parameter is not predefined in the initial configuration files. You can manually set it if you need it. The system default value is UTC+00:00:00. For the format of the parameter value, see Specifying the Time Zone with TZ. For example, --timezone_name=UTC+08:00 represents the GMT+8 time zone.\nNo\nLogging configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nlog_dir\nlogs\nThe directory that stores the Meta Service log. It is recommended to put logs on a different hard disk from the data.\nNo\nminloglevel\nSpecifies the minimum level of the log. That is, log messages at or above this level. Optional values are 0 (INFO), 1 (WARNING), 2 (ERROR), 3 (FATAL). It is recommended to set it to 0 during debugging and 1 in a production environment. If it is set to 4, NebulaGraph will not print any logs.\nYes\nSpecifies the detailed level of VLOG. That is, log all VLOG messages less or equal to the level. Optional values are 0, 1, 2, 3, 4, 5. The VLOG macro provided by glog allows users to define their own numeric logging levels and control verbose messages that are logged with the parameter v. For details, see Verbose Logging.\nYes\nlogbufsecs\nSpecifies the maximum time to buffer the logs. If there is a timeout, it will output the buffered log to the log file. 0 means real-time output. This configuration is measured in seconds.\nNo\nredirect_stdout\ntrue\nWhen set to true, the process redirects thestdout and stderr to separate output files.\nNo\nstdout_log_file\nmetad-stdout.log\nSpecifies the filename for the stdout log.\nNo\nstderr_log_file\nmetad-stderr.log\nSpecifies the filename for the stderr log.\nNo\nstderrthreshold\nSpecifies the minloglevel to be copied to the stderr log.\nNo\ntimestamp_in_logfile_name\ntrue\nSpecifies if the log file name contains a timestamp. true indicates yes, false indicates no.\nNo\nNetworking configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nmeta_server_addrs\n127.0.0.1:9559\nSpecifies the IPs (or hostnames) and ports of all Meta Services. Multiple addresses are separated with commas.\nNo\nlocal_ip\n127.0.0.1\nSpecifies the local IP (or hostname) for the Meta Service. The local IP address is used to identify the nebula-metad process. If it is a distributed cluster or requires remote access, modify it to the corresponding address.\nNo\nport\n9559\nSpecifies RPC daemon listening port of the Meta service. The neighboring +1 (9560) port is used for Raft communication between Meta services.\nNo\nws_ip\n0.0.0.0\nSpecifies the IP address for the HTTP service.\nNo\nws_http_port\n19559\nSpecifies the port for the HTTP service.\nNo\nws_storage_http_port\n19779\nSpecifies the Storage service listening port used by the HTTP protocol. It must be consistent with the ws_http_port in the Storage service configuration file. This parameter only applies to standalone NebulaGraph.\nNo\nheartbeat_interval_secs\n10\nSpecifies the default heartbeat interval. Make sure the heartbeat_interval_secs values for all services are the same, otherwise NebulaGraph CANNOT work normally. This configuration is measured in seconds.\nYes\nStorage configurations\nName\nPredefined Value\nDescription\nWhether supports runtime dynamic modifications\ndata_path\ndata/meta\nThe storage path for Meta data.\nNo\nMisc configurations\nName\nPredefined Value\nDescription\nWhether supports runtime dynamic modifications\ndefault_parts_num\n10\nSpecifies the default partition number when creating a new graph space.\nNo\ndefault_replica_factor\nSpecifies the default replica number when creating a new graph space.\nNo\nRocksDB options configurations\nName\nPredefined Value\nDescription\nWhether supports runtime dynamic modifications\nrocksdb_wal_sync\ntrue\nEnables or disables RocksDB WAL synchronization. Available values are true (enable) and false (disable).\nNo","url":"https://docs.nebula-graph.io/3.6.0/5.configurations-and-logs/1.configurations/2.meta-config/","type":"doc"},{"title":"Graph Service configuration","content":"NebulaGraph provides two initial configuration files for the Graph Service, nebula-graphd.conf.default and nebula-graphd.conf.production. Users can use them in different scenarios conveniently. The default file path is /usr/local/nebula/etc/.\nHow to use the configuration files\nTo use the initial configuration file, choose one of the above two files and delete the suffix .default or .production from the initial configuration file for the Meta Service to apply the configurations defined in it.\nAbout parameter values\nIf a parameter is not set in the configuration file, NebulaGraph uses the default value. Not all parameters are predefined. And the predefined parameters in the two initial configuration files are different. This topic uses the parameters in nebula-metad.conf.default.\nFor all parameters and their current values, see Configurations.\nBasics configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\ndaemonize\ntrue\nWhen set to true, the process is a daemon process.\nNo\npid_file\npids/nebula-graphd.pid\nThe file that records the process ID.\nNo\nenable_optimizer\ntrue\nWhen set to true, the optimizer is enabled.\nNo\ntimezone_name\nSpecifies the NebulaGraph time zone. This parameter is not predefined in the initial configuration files. The system default value is UTC+00:00:00. For the format of the parameter value, see Specifying the Time Zone with TZ. For example, --timezone_name=UTC+08:00 represents the GMT+8 time zone.\nNo\nlocal_config\ntrue\nWhen set to true, the process gets configurations from the configuration files.\nNo\nLogging configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nlog_dir\nlogs\nThe directory that stores the Meta Service log. It is recommended to put logs on a different hard disk from the data.\nNo\nminloglevel\nSpecifies the minimum level of the log. That is, log messages at or above this level. Optional values are 0 (INFO), 1 (WARNING), 2 (ERROR), 3 (FATAL). It is recommended to set it to 0 during debugging and 1 in a production environment. If it is set to 4, NebulaGraph will not print any logs.\nYes\nSpecifies the detailed level of VLOG. That is, log all VLOG messages less or equal to the level. Optional values are 0, 1, 2, 3, 4, 5. The VLOG macro provided by glog allows users to define their own numeric logging levels and control verbose messages that are logged with the parameter v. For details, see Verbose Logging.\nYes\nlogbufsecs\nSpecifies the maximum time to buffer the logs. If there is a timeout, it will output the buffered log to the log file. 0 means real-time output. This configuration is measured in seconds.\nNo\nredirect_stdout\ntrue\nWhen set to true, the process redirects thestdout and stderr to separate output files.\nNo\nstdout_log_file\ngraphd-stdout.log\nSpecifies the filename for the stdout log.\nNo\nstderr_log_file\ngraphd-stderr.log\nSpecifies the filename for the stderr log.\nNo\nstderrthreshold\nSpecifies the minloglevel to be copied to the stderr log.\nNo\ntimestamp_in_logfile_name\ntrue\nSpecifies if the log file name contains a timestamp. true indicates yes, false indicates no.\nNo\nQuery configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\naccept_partial_success\nfalse\nWhen set to false, the process treats partial success as an error. This configuration only applies to read-only requests. Write requests always treat partial success as an error. A partial success query will prompt Got partial result.\nYes\nsession_reclaim_interval_secs\n60\nSpecifies the interval that the Session information is sent to the Meta service. This configuration is measured in seconds.\nYes\nmax_allowed_query_size\n4194304\nSpecifies the maximum length of queries. Unit: bytes. The default value is 4194304, namely 4MB.\nYes\nNetworking configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nmeta_server_addrs\n127.0.0.1:9559\nSpecifies the IPs (or hostnames) and ports of all Meta Services. Multiple addresses are separated with commas.\nNo\nlocal_ip\n127.0.0.1\nSpecifies the local IP (or hostname) for the Graph Service. The local IP address is used to identify the nebula-graphd process. If it is a distributed cluster or requires remote access, modify it to the corresponding address.\nNo\nlisten_netdev\nany\nSpecifies the listening network device.\nNo\nport\n9669\nSpecifies RPC daemon listening port of the Graph service.\nNo\nreuse_port\nfalse\nWhen set to false, the SO_REUSEPORT is closed.\nNo\nlisten_backlog\n1024\nSpecifies the maximum length of the connection queue for socket monitoring. This configuration must be modified together with the net.core.somaxconn.\nNo\nclient_idle_timeout_secs\n28800\nSpecifies the time to expire an idle connection. The value ranges from 1 to 604800. The default is 8 hours. This configuration is measured in seconds.\nNo\nsession_idle_timeout_secs\n28800\nSpecifies the time to expire an idle session. The value ranges from 1 to 604800. The default is 8 hours. This configuration is measured in seconds.\nNo\nnum_accept_threads\nSpecifies the number of threads that accept incoming connections.\nNo\nnum_netio_threads\nSpecifies the number of networking IO threads. 0 is the number of CPU cores.\nNo\nnum_max_connections\nMax active connections for all networking threads. 0 means no limit.Max connections for each networking thread = num_max_connections / num_netio_threads\nNo\nnum_worker_threads\nSpecifies the number of threads that execute queries. 0 is the number of CPU cores.\nNo\nws_ip\n0.0.0.0\nSpecifies the IP address for the HTTP service.\nNo\nws_http_port\n19669\nSpecifies the port for the HTTP service.\nNo\nheartbeat_interval_secs\n10\nSpecifies the default heartbeat interval. Make sure the heartbeat_interval_secs values for all services are the same, otherwise NebulaGraph CANNOT work normally. This configuration is measured in seconds.\nYes\nstorage_client_timeout_ms\nSpecifies the RPC connection timeout threshold between the Graph Service and the Storage Service. This parameter is not predefined in the initial configuration files. You can manually set it if you need it. The system default value is 60000 ms.\nNo\nslow_query_threshold_us\n200000\nWhen the execution time of a query exceeds the value, the query is called a slow query. Unit: Microsecond.Note: Even if the execution time of DML statements exceeds this value, they will not be recorded as slow queries.\nNo\nws_meta_http_port\n19559\nSpecifies the Meta service listening port used by the HTTP protocol. It must be consistent with the ws_http_port in the Meta service configuration file.\nNo\nCharset and collate configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\ndefault_charset\nutf8\nSpecifies the default charset when creating a new graph space.\nNo\ndefault_collate\nutf8_bin\nSpecifies the default collate when creating a new graph space.\nNo\nAuthorization configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nenable_authorize\nfalse\nWhen set to false, the system authentication is not enabled. For more information, see Authentication.\nNo\nauth_type\npassword\nSpecifies the login method. Available values are password, ldap, and cloud.\nNo\nMemory configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nsystem_memory_high_watermark_ratio\n0.8\nSpecifies the trigger threshold of the high-level memory alarm mechanism. If the system memory usage is higher than this value, an alarm mechanism will be triggered, and NebulaGraph will stop querying. This parameter is not predefined in the initial configuration files.\nYes\nMetrics configurations\nSession configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nmax_sessions_per_ip_per_user\n300\nThe maximum number of active sessions that can be created from a single IP adddress for a single user.\nNo\nExperimental configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nenable_experimental_feature\nfalse\nSpecifies the experimental feature. Optional values are true and false.\nNo\nenable_data_balance\ntrue\nWhether to enable the BALANCE DATA feature. Only works when enable_experimental_feature is true.\nNo\nMemory tracker configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nmemory_tracker_limit_ratio\n0.8\nThe value of this parameter can be set to (0, 1], 2, and 3.Caution: When setting this parameter, ensure that the value of system_memory_high_watermark_ratio is not set to 1, otherwise the value of this parameter will not take effect.(0, 1]: The percentage of free memory. When the free memory is lower than this value, NebulaGraph stops accepting queries. Calculated as follows: Free memory / (Total memory - Reserved memory) Note: For clusters with a mixed-used environment, the value of memory_tracker_limit_ratio should be set to a lower value. For example, when Graphd is expected to occupy only 50% of memory, the value can be set to less than 0.5.2: Dynamic Self Adaptive mode. MemoryTracker dynamically adjusts the available memory based on the system's current available memory. Note: This feature is experimental. As memory usage cannot be monitored in real time in dynamic adaptive mode, an OOM error may still occur to handle large memory allocations. 3: Disable MemoryTracker. MemoryTracker only logs memory usage and does not interfere with executions even if the limit is exceeded.\nYes\nmemory_tracker_untracked_reserved_memory_mb\n50\nThe reserved memory that is not tracked by the memory tracker. Unit: MB.\nYes\nmemory_tracker_detail_log\nfalse\nWhether to enable the memory tracker log. When the value is true, the memory tracker log is generated.\nYes\nmemory_tracker_detail_log_interval_ms\n60000\nThe time interval for generating the memory tracker log. Unit: Millisecond. memory_tracker_detail_log is true when this parameter takes effect.\nYes\nmemory_purge_enabled\ntrue\nWhether to enable the memory purge feature. When the value is true, the memory purge feature is enabled.\nYes\nmemory_purge_interval_seconds\n10\nThe time interval for the memory purge feature to purge memory. Unit: Second. This parameter only takes effect if memory_purge_enabled is set to true.\nYes\nperformance optimization configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nmax_job_size\nThe maximum number of concurrent jobs, i.e., the maximum number of threads used in the phase of query execution where concurrent execution is possible. It is recommended to be half of the physical CPU cores.\nYes\nmin_batch_size\n8192\nThe minimum batch size for processing the dataset. Takes effect only when max_job_size is greater than 1.\nYes\noptimize_appendvertices\nfalse\nWhen enabled, the MATCH statement is executed without filtering dangling edges.\nYes\npath_batch_size\n10000\nThe number of paths constructed per thread.\nYes","url":"https://docs.nebula-graph.io/3.6.0/5.configurations-and-logs/1.configurations/3.graph-config/","type":"doc"},{"title":"Storage Service configurations","content":"NebulaGraph provides two initial configuration files for the Storage Service, nebula-storaged.conf.default and nebula-storaged.conf.production. Users can use them in different scenarios conveniently. The default file path is /usr/local/nebula/etc/.\nHow to use the configuration files\nTo use the initial configuration file, choose one of the above two files and delete the suffix .default or .production from the initial configuration file for the Meta Service to apply the configurations defined in it.\nAbout parameter values\nIf a parameter is not set in the configuration file, NebulaGraph uses the default value. Not all parameters are predefined. And the predefined parameters in the two initial configuration files are different. This topic uses the parameters in nebula-metad.conf.default. For parameters that are not included in nebula-metad.conf.default, see nebula-storaged.conf.production.\nFor all parameters and their current values, see Configurations.\nBasics configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\ndaemonize\ntrue\nWhen set to true, the process is a daemon process.\nNo\npid_file\npids/nebula-storaged.pid\nThe file that records the process ID.\nNo\ntimezone_name\nUTC+00:00:00\nSpecifies the NebulaGraph time zone. This parameter is not predefined in the initial configuration files, if you need to use this parameter, add it manually. For the format of the parameter value, see Specifying the Time Zone with TZ. For example, --timezone_name=UTC+08:00 represents the GMT+8 time zone.\nNo\nlocal_config\ntrue\nWhen set to true, the process gets configurations from the configuration files.\nNo\nLogging configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nlog_dir\nlogs\nThe directory that stores the Meta Service log. It is recommended to put logs on a different hard disk from the data.\nNo\nminloglevel\nSpecifies the minimum level of the log. That is, log messages at or above this level. Optional values are 0 (INFO), 1 (WARNING), 2 (ERROR), 3 (FATAL). It is recommended to set it to 0 during debugging and 1 in a production environment. If it is set to 4, NebulaGraph will not print any logs.\nYes\nSpecifies the detailed level of VLOG. That is, log all VLOG messages less or equal to the level. Optional values are 0, 1, 2, 3, 4, 5. The VLOG macro provided by glog allows users to define their own numeric logging levels and control verbose messages that are logged with the parameter v. For details, see Verbose Logging.\nYes\nlogbufsecs\nSpecifies the maximum time to buffer the logs. If there is a timeout, it will output the buffered log to the log file. 0 means real-time output. This configuration is measured in seconds.\nNo\nredirect_stdout\ntrue\nWhen set to true, the process redirects thestdout and stderr to separate output files.\nNo\nstdout_log_file\ngraphd-stdout.log\nSpecifies the filename for the stdout log.\nNo\nstderr_log_file\ngraphd-stderr.log\nSpecifies the filename for the stderr log.\nNo\nstderrthreshold\nSpecifies the minloglevel to be copied to the stderr log.\nNo\ntimestamp_in_logfile_name\ntrue\nSpecifies if the log file name contains a timestamp. true indicates yes, false indicates no.\nNo\nNetworking configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nmeta_server_addrs\n127.0.0.1:9559\nSpecifies the IPs (or hostnames) and ports of all Meta Services. Multiple addresses are separated with commas.\nNo\nlocal_ip\n127.0.0.1\nSpecifies the local IP (or hostname) for the Storage Service. The local IP address is used to identify the nebula-storaged process. If it is a distributed cluster or requires remote access, modify it to the corresponding address.\nNo\nport\n9779\nSpecifies RPC daemon listening port of the Storage service. The neighboring ports -1 (9778) and +1 (9780) are also used. 9778: The port used by the Admin service, which receives Meta commands for Storage. 9780: The port used for Raft communication between Storage services.\nNo\nws_ip\n0.0.0.0\nSpecifies the IP address for the HTTP service.\nNo\nws_http_port\n19779\nSpecifies the port for the HTTP service.\nNo\nheartbeat_interval_secs\n10\nSpecifies the default heartbeat interval. Make sure the heartbeat_interval_secs values for all services are the same, otherwise NebulaGraph CANNOT work normally. This configuration is measured in seconds.\nYes\nRaft configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nraft_heartbeat_interval_secs\n30\nSpecifies the time to expire the Raft election. The configuration is measured in seconds.\nYes\nraft_rpc_timeout_ms\n500\nSpecifies the time to expire the Raft RPC. The configuration is measured in milliseconds.\nYes\nwal_ttl\n14400\nSpecifies the lifetime of the RAFT WAL. The configuration is measured in seconds.\nYes\nDisk configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\ndata_path\ndata/storage\nSpecifies the data storage path. Multiple paths are separated with commas. For NebulaGraph of the community edition, one RocksDB instance corresponds to one path.\nNo\nminimum_reserved_bytes\n268435456\nSpecifies the minimum remaining space of each data storage path. When the value is lower than this standard, the cluster data writing may fail. This configuration is measured in bytes.\nNo\nrocksdb_batch_size\n4096\nSpecifies the block cache for a batch operation. The configuration is measured in bytes.\nNo\nrocksdb_block_cache\nSpecifies the block cache for BlockBasedTable. The configuration is measured in megabytes.\nNo\ndisable_page_cache\nfalse\nEnables or disables the operating system's page cache for NebulaGraph. By default, the parameter value is false and page cache is enabled. If the value is set to true, page cache is disabled and sufficient block cache space must be configured for NebulaGraph.\nNo\nengine_type\nrocksdb\nSpecifies the engine type.\nNo\nrocksdb_compression\nlz4\nSpecifies the compression algorithm for RocksDB. Optional values are no, snappy, lz4, lz4hc, zlib, bzip2, and zstd.This parameter modifies the compression algorithm for each level. If you want to set different compression algorithms for each level, use the parameter rocksdb_compression_per_level.\nNo\nrocksdb_compression_per_level\nSpecifies the compression algorithm for each level. The priority is higher than rocksdb_compression. For example, no:no:lz4:lz4:snappy:zstd:snappy.You can also not set certain levels of compression algorithms, for example, no:no:lz4:lz4::zstd, level L4 and L6 use the compression algorithm of rocksdb_compression.\nNo\nenable_rocksdb_statistics\nfalse\nWhen set to false, RocksDB statistics is disabled.\nNo\nrocksdb_stats_level\nkExceptHistogramOrTimers\nSpecifies the stats level for RocksDB. Optional values are kExceptHistogramOrTimers, kExceptTimers, kExceptDetailedTimers, kExceptTimeForMutex, and kAll.\nNo\nenable_rocksdb_prefix_filtering\ntrue\nWhen set to true, the prefix bloom filter for RocksDB is enabled. Enabling prefix bloom filter makes the graph traversal faster but occupies more memory.\nNo\nenable_rocksdb_whole_key_filtering\nfalse\nWhen set to true, the whole key bloom filter for RocksDB is enabled.\nrocksdb_filtering_prefix_length\n12\nSpecifies the prefix length for each key. Optional values are 12 and 16. The configuration is measured in bytes.\nNo\nenable_partitioned_index_filter\nfalse\nWhen set to true, it reduces the amount of memory used by the bloom filter. But in some random-seek situations, it may reduce the read performance. This parameter is not predefined in the initial configuration files, if you need to use this parameter, add it manually.\nNo\nMisc configurations\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nquery_concurrently\ntrue\nWhether to turn on multi-threaded queries. Enabling it can improve the latency performance of individual queries, but it will reduce the overall throughput under high pressure.\nYes\nauto_remove_invalid_space\ntrue\nAfter executing DROP SPACE, the specified graph space will be deleted. This parameter sets whether to delete all the data in the specified graph space at the same time. When the value is true, all the data in the specified graph space will be deleted at the same time.\nYes\nnum_io_threads\n16\nThe number of network I/O threads used to send RPC requests and receive responses.\nNo\nnum_max_connections\nMax active connections for all networking threads. 0 means no limit.Max connections for each networking thread = num_max_connections / num_netio_threads\nNo\nnum_worker_threads\n32\nThe number of worker threads for one RPC-based Storage service.\nNo\nmax_concurrent_subtasks\n10\nThe maximum number of concurrent subtasks to be executed by the task manager.\nNo\nsnapshot_part_rate_limit\n10485760\nThe rate limit when the Raft leader synchronizes the stock data with other members of the Raft group. Unit: bytes/s.\nYes\nsnapshot_batch_size\n1048576\nThe amount of data sent in each batch when the Raft leader synchronizes the stock data with other members of the Raft group. Unit: bytes.\nYes\nrebuild_index_part_rate_limit\n4194304\nThe rate limit when the Raft leader synchronizes the index data rate with other members of the Raft group during the index rebuilding process. Unit: bytes/s.\nYes\nrebuild_index_batch_size\n1048576\nThe amount of data sent in each batch when the Raft leader synchronizes the index data with other members of the Raft group during the index rebuilding process. Unit: bytes.\nYes\nRocksDB options\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nrocksdb_db_options\n{}\nSpecifies the RocksDB database options.\nNo\nrocksdb_column_family_options\n{\"write_buffer_size\":\"67108864\",\"max_write_buffer_number\":\"4\",\"max_bytes_for_level_base\":\"268435456\"}\nSpecifies the RocksDB column family options.\nNo\nrocksdb_block_based_table_options\n{\"block_size\":\"8192\"}\nSpecifies the RocksDB block based table options.\nNo\nThe format of the RocksDB option is {\"\":\"\"}. Multiple options are separated with commas.\nSupported options of rocksdb_db_options and rocksdb_column_family_options are listed as follows.\nrocksdb_db_optionsmax_total_wal_size\ndelete_obsolete_files_period_micros\nmax_background_jobs\nstats_dump_period_sec\ncompaction_readahead_size\nwritable_file_max_buffer_size\nbytes_per_sync\nwal_bytes_per_sync\ndelayed_write_rate\navoid_flush_during_shutdown\nmax_open_files\nstats_persist_period_sec\nstats_history_buffer_size\nstrict_bytes_per_sync\nenable_rocksdb_prefix_filtering\nenable_rocksdb_whole_key_filtering\nrocksdb_filtering_prefix_length\nnum_compaction_threads\nrate_limit\nrocksdb_column_family_optionswrite_buffer_size\nmax_write_buffer_number\nlevel0_file_num_compaction_trigger\nlevel0_slowdown_writes_trigger\nlevel0_stop_writes_trigger\ntarget_file_size_base\ntarget_file_size_multiplier\nmax_bytes_for_level_base\nmax_bytes_for_level_multiplier\ndisable_auto_compactions \nFor more information, see RocksDB official documentation.\nMemory Tracker configurations\nFor details about Memory Tracker, see Memory Tracker: Memory Management Practice in NebulaGraph Database\nName\nPredefined value\nDescription\nWhether supports runtime dynamic modifications\nmemory_tracker_limit_ratio\n0.8\nThe value of this parameter can be set to (0, 1], 2, and 3.(0, 1]: The percentage of free memory. When the free memory is lower than this value, NebulaGraph stops accepting queries. Calculated as follows: Free memory / (Total memory - Reserved memory) Note: For clusters with a mixed-used environment, the value of memory_tracker_limit_ratio should be set to a lower value. For example, when Graphd is expected to occupy only 50% of memory, the value can be set to less than 0.5.2: Dynamic Self Adaptive mode. MemoryTracker dynamically adjusts the available memory based on the system's current available memory. Note: This feature is experimental. As memory usage cannot be monitored in real time in dynamic adaptive mode, an OOM error may still occur to handle large memory allocations. 3: Disable MemoryTracker. MemoryTracker only logs memory usage and does not interfere with executions even if the limit is exceeded.\nYes\nmemory_tracker_untracked_reserved_memory_mb\n50\nThe reserved memory that is not tracked by the Memory Tracker. Unit: MB.\nYes\nmemory_tracker_detail_log\nfalse\nWhether to enable the Memory Tracker log. When the value is true, the Memory Tracker log is generated.\nYes\nmemory_tracker_detail_log_interval_ms\n60000\nThe time interval for generating the Memory Tracker log. Unit: Millisecond. memory_tracker_detail_log is true when this parameter takes effect.\nYes\nmemory_purge_enabled\ntrue\nWhether to enable the memory purge feature. When the value is true, the memory purge feature is enabled.\nYes\nmemory_purge_interval_seconds\n10\nThe time interval for the memory purge feature to purge memory. Unit: Second. This parameter only takes effect if memory_purge_enabled is set to true.\nYes\nFor super-Large vertices\nWhen the query starting from each vertex gets an edge, truncate it directly to avoid too many neighboring edges on the super-large vertex, because a single query occupies too much hard disk and memory. Or you can truncate a certain number of edges specified in the Max_edge_returned_per_vertex parameter. Excess edges will not be returned. This parameter applies to all spaces.\nProperty name\nDefault value\nDescription\nWhether supports runtime dynamic modifications\nmax_edge_returned_per_vertex\n2147483647\nSpecifies the maximum number of edges returned for each dense vertex. Excess edges are truncated and not returned. This parameter is not predefined in the initial configuration files, if you need to use this parameter, add it manually.\nNo\nStorage configurations for large dataset\nWhen you have a large dataset (in the RocksDB directory) and your memory is tight, we suggest that you set the enable_partitioned_index_filter parameter to true. The performance is affected because RocksDB indexes are cached.","url":"https://docs.nebula-graph.io/3.6.0/5.configurations-and-logs/1.configurations/4.storage-config/","type":"doc"},{"title":"Kernel configurations","content":"This topic introduces the Kernel configurations in Nebula Graph.\nResource control\nYou may run the ulimit command to control the resource threshold. However, the changes made only take effect for the current session or sub-process. To make permanent changes, edit file /etc/security/limits.conf. The configuration is as follows:\n# \n* soft core unlimited \n* hard core unlimited \n* soft nofile 130000 \n* hard nofile 130000\nThe parameter descriptions are as follows.\nParameter\nDescription\ndomain\nControl Domain. This parameter can be a user name, a user group name (starting with @), or * to indicate all users.\ntype\nControl type. This parameter can be soft or hard. soft indicates a soft threshold (the default threshold) for the resource and hard indicates a maximum value that can be set by the user. The ulimit command can be used to increase soft, but not to exceed hard.\nitem\nResource types. For example, core limits the size of the core dump file, and nofile limits the maximum number of file descriptors a process can open.\nvalue\nResource limit value. This parameter can be a number, or unlimited to indicate that there is no limit.\nYou can run man limits.conf for more helpful information.\nMemory\nvm.swappiness\nvm.swappiness specifies the percentage of the available memory before starting swap. The greater the value, the more likely the swap occurs. We recommend that you set it to 0. When set to 0, the page cache is removed first. Note that when vm.swappiness is 0, it does not mean that there is no swap.\nvm.min_free_kbytes\nvm.min_free_kbytes specifies the minimum number of kilobytes available kept by Linux VM. If you have a large system memory, we recommend that you increase this value. For example, if your physical memory 128GB, set it to 5GB. If the value is not big enough, the system cannot apply for enough continuous physical memory.\nvm.max_map_count\nvm.max_map_count limits the maximum number of vma (virtual memory area) for a process. The default value is 65530. It is enough for most applications. If your memory application fails because the memory consumption is large, increase the vm.max_map_count value.\nvm.dirty_*\nThese values control the dirty data cache for the system. For write-intensive scenarios, you can make adjustments based on your needs (throughput priority or delay priority). We recommend that you use the system default value.\nTransparent huge page\nFor better delay performance, you must run the following commands to disable the transparent huge pages (THP).\nroot# echo never > /sys/kernel/mm/transparent_hugepage/enabled\nroot# echo never > /sys/kernel/mm/transparent_hugepage/defrag\nroot# swapoff -a && swapon -a\nTo prevent THP from being enabled again after the system restarts, you can modify the GRUB configuration file or /etc/rc.local to disable THP automatically upon system startup.\nNetworking\nnet.ipv4.tcp_slow_start_after_idle\nThe default value of net.ipv4.tcp_slow_start_after_idle is 1. If set, the congestion window is timed out after an idle period. We recommend that you set it to 0, especially for long fat scenarios (high latency and large bandwidth).\nnet.core.somaxconn\nnet.core.somaxconn specifies the maximum number of connection queues listened by the socket. The default value is 128. For scenarios with a large number of burst connections, we recommend that you set it to greater than 1024.\nnet.ipv4.tcp_max_syn_backlog\nnet.ipv4.tcp_max_syn_backlog specifies the maximum number of TCP connections in the SYN_RECV (semi-connected) state. The setting rule for this parameter is the same as that of net.core.somaxconn.\nnet.core.netdev_max_backlog\nnet.core.netdev_max_backlog specifies the maximum number of packets. The default value is 1000. We recommend that you increase it to greater than 10,000, especially for 10G network adapters.\nnet.ipv4.tcp_keepalive_*\nThese values keep parameters alive for TCP connections. For applications that use a 4-layer transparent load balancer, if the idle connection is disconnected unexpectedly, decrease the values of tcp_keepalive_time and tcp_keepalive_intvl.\nnet.ipv4.tcp_rmem/wmem\nnet.ipv4.tcp_wmem/rmem specifies the minimum, default, and maximum size of the buffer pool sent/received by the TCP socket. For long fat links, we recommend that you increase the default value to bandwidth (GB) * RTT (ms).\nscheduler\nFor SSD devices, we recommend that you set scheduler to noop or none. The path is /sys/block/DEV_NAME/queue/scheduler.\nOther parameters\nkernel.core_pattern\nwe recommend that you set it to core and set kernel.core_uses_pid to 1.\nModify parameters\nsysctl\nsysctl \nChecks the current parameter value.\nsysctl -w =\nModifies the parameter value. The modification takes effect immediately. The original value is restored after restarting.\nsysctl -p []  \nLoads Linux parameter values ​​from the specified configuration file. The default path is /etc/sysctl.conf.\nprlimit\nThe prlimit command gets and sets process resource limits. You can modify the hard threshold by using it and the sudo command. For example, prlimit --nofile = 130000 --pid = $$ adjusts the maximum number of open files permitted by the current process to 14000. And the modification takes effect immediately. Note that this command is only available in RedHat 7u or higher versions.","url":"https://docs.nebula-graph.io/3.6.0/5.configurations-and-logs/1.configurations/6.kernel-config/","type":"doc"},{"title":"Runtime logs","content":"Runtime logs are provided for DBAs and developers to locate faults when the system fails.\nNebulaGraph uses glog to print runtime logs, uses gflags to control the severity level of the log, and provides an HTTP interface to dynamically change the log level at runtime to facilitate tracking.\nLog directory\nThe default runtime log directory is /usr/local/nebula/logs/.\nIf the log directory is deleted while NebulaGraph is running, the log would not continue to be printed. However, this operation will not affect the services. To recover the logs, restart the services.\nParameter descriptions\nminloglevel: Specifies the minimum level of the log. That is, no logs below this level will be printed. Optional values are 0 (INFO), 1 (WARNING), 2 (ERROR), 3 (FATAL). It is recommended to set it to 0 during debugging and 1 in a production environment. If it is set to 4, NebulaGraph will not print any logs.\nv: Specifies the detailed level of the log. The larger the value, the more detailed the log is. Optional values are 0, 1, 2, 3.\nThe default severity level for the metad, graphd, and storaged logs can be found in their respective configuration files. The default path is /usr/local/nebula/etc/.\nCheck the severity level\nCheck all the flag values (log values included) of the current gflags with the following command.\n$ curl :/flags\nParameter\nDescription\nws_ip\nThe IP address for the HTTP service, which can be found in the configuration files above. The default value is 127.0.0.1.\nws_port\nThe port for the HTTP service, which can be found in the configuration files above. The default values are 19559(Meta), 19669(Graph), and 19779(Storage) respectively.\nExamples are as follows:\nCheck the current minloglevel in the Meta service:$ curl 127.0.0.1:19559/flags | grep 'minloglevel'\nCheck the current v in the Storage service:$ curl 127.0.0.1:19779/flags | grep -w 'v'\nChange the severity level\nChange the severity level of the log with the following command.\n$ curl -X PUT -H \"Content-Type: application/json\" -d '{\"\":[,\"\":]}' \":/flags\"\nParameter\nDescription\nkey\nThe type of the log to be changed. For optional values, see Parameter descriptions.\nvalue\nThe level of the log. For optional values, see Parameter descriptions.\nws_ip\nThe IP address for the HTTP service, which can be found in the configuration files above. The default value is 127.0.0.1.\nws_port\nThe port for the HTTP service, which can be found in the configuration files above. The default values are 19559(Meta), 19669(Graph), and 19779(Storage) respectively.\nExamples are as follows:\n$ curl -X PUT -H \"Content-Type: application/json\" -d '{\"minloglevel\":0,\"v\":3}' \"127.0.0.1:19779/flags\" # storaged\n$ curl -X PUT -H \"Content-Type: application/json\" -d '{\"minloglevel\":0,\"v\":3}' \"127.0.0.1:19669/flags\" # graphd\n$ curl -X PUT -H \"Content-Type: application/json\" -d '{\"minloglevel\":0,\"v\":3}' \"127.0.0.1:19559/flags\" # metad\nIf the log level is changed while NebulaGraph is running, it will be restored to the level set in the configuration file after restarting the service. To permanently modify it, see Configuration files.\nRocksDB runtime logs\nRocksDB runtime logs are usually used to debug RocksDB parameters and stored in /usr/local/nebula/data/storage/nebula/$id/data/LOG. $id is the ID of the example.\nLog recycle\nHow to recycle logs when the runtime log file is too large, see FAQs.","url":"https://docs.nebula-graph.io/3.6.0/5.configurations-and-logs/2.log-management/logs/","type":"doc"},{"title":"Query NebulaGraph metrics","content":"NebulaGraph supports querying the monitoring metrics through HTTP ports.\nMetrics structure\nEach metric of NebulaGraph consists of three fields: name, type, and time range. The fields are separated by periods, for example, num_queries.sum.600. Different NebulaGraph services (Graph, Storage, or Meta) support different metrics. The detailed description is as follows.\nField\nExample\nDescription\nMetric name\nnum_queries\nIndicates the function of the metric.\nMetric type\nsum\nIndicates how the metrics are collected. Supported types are SUM, AVG, RATE, and the P-th sample quantiles such as P75, P95, P99, and P99.9.\nTime range\n600\nThe time range in seconds for the metric collection. Supported values are 5, 60, 600, and 3600, representing the last 5 seconds, 1 minute, 10 minutes, and 1 hour.\nQuery metrics over HTTP\nSyntax\ncurl -G \"http://:/stats?stats= [&format=json]\"\nParameter\nDescription\nhost\nThe IP (or hostname) of the server. You can find it in the configuration file in the installation directory.\nport\nThe HTTP port of the server. You can find it in the configuration file in the installation directory. The default ports are 19559 (Meta), 19669 (Graph), and 19779 (Storage).\nmetric_name_list\nThe metrics names. Multiple metrics are separated by commas (,).\n&format=json\nOptional. Returns the result in the JSON format.\nQuery a single metric\nQuery the query number in the last 10 minutes in the Graph Service.\n$ curl -G \"http://192.168.8.40:19669/stats?stats=num_queries.sum.600\"\nnum_queries.sum.600=400\nQuery multiple metrics\nQuery the following metrics together:\nThe average heartbeat latency in the last 1 minute.\nThe average latency of the slowest 1% heartbeats, i.e., the P99 heartbeats, in the last 10 minutes.\n$ curl -G \"http://192.168.8.40:19559/stats?stats=heartbeat_latency_us.avg.60,heartbeat_latency_us.p99.600\"\nheartbeat_latency_us.avg.60=281\nheartbeat_latency_us.p99.600=985\nReturn a JSON result.\nQuery the number of new vertices in the Storage Service in the last 10 minutes and return the result in the JSON format.\n$ curl -G \"http://192.168.8.40:19779/stats?stats=num_add_vertices.sum.600&format=json\"\n[{\"value\":1,\"name\":\"num_add_vertices.sum.600\"}]\nQuery all metrics in a service.\nIf no metric is specified in the query, NebulaGraph returns all metrics in the service.\n$ curl -G \"http://192.168.8.40:19559/stats\"\nheartbeat_latency_us.avg.5=304\nheartbeat_latency_us.avg.60=308\nheartbeat_latency_us.avg.600=299\nheartbeat_latency_us.avg.3600=285\nheartbeat_latency_us.p75.5=652\nheartbeat_latency_us.p75.60=669\nheartbeat_latency_us.p75.600=651\nheartbeat_latency_us.p75.3600=642\nheartbeat_latency_us.p95.5=930\nheartbeat_latency_us.p95.60=963\nheartbeat_latency_us.p95.600=933\nheartbeat_latency_us.p95.3600=929\nheartbeat_latency_us.p99.5=986\nheartbeat_latency_us.p99.60=1409\nheartbeat_latency_us.p99.600=989\nheartbeat_latency_us.p99.3600=986\nnum_heartbeats.rate.5=0\nnum_heartbeats.rate.60=0\nnum_heartbeats.rate.600=0\nnum_heartbeats.rate.3600=0\nnum_heartbeats.sum.5=2\nnum_heartbeats.sum.60=40\nnum_heartbeats.sum.600=394\nnum_heartbeats.sum.3600=2364\n...\nSpace-level metrics\nThe Graph service supports a set of space-level metrics that record the information of different graph spaces separately.\nSpace-level metrics can be queried only by querying all metrics. For example, run curl -G \"http://192.168.8.40:19559/stats\" to show all metrics. The returned result contains the graph space name in the form of '{space=space_name}', such as num_active_queries{space=basketballplayer}.sum.5=0.\nMetric description\nGraph\nParameter\nDescription\nnum_active_queries\nThe number of changes in the number of active queries. Formula: The number of started queries minus the number of finished queries within a specified time.\nnum_active_sessions\nThe number of changes in the number of active sessions. Formula: The number of logged in sessions minus the number of logged out sessions within a specified time.For example, when querying num_active_sessions.sum.5, if there were 10 sessions logged in and 30 sessions logged out in the last 5 seconds, the value of this metric is -20 (10-30).\nnum_aggregate_executors\nThe number of executions for the Aggregation operator.\nnum_auth_failed_sessions_bad_username_password\nThe number of sessions where authentication failed due to incorrect username and password.\nnum_auth_failed_sessions_out_of_max_allowed\nThe number of sessions that failed to authenticate logins because the value of the parameter FLAG_OUT_OF_MAX_ALLOWED_CONNECTIONS was exceeded.\nnum_auth_failed_sessions\nThe number of sessions in which login authentication failed.\nnum_indexscan_executors\nThe number of executions for index scan operators.\nnum_killed_queries\nThe number of killed queries.\nnum_opened_sessions\nThe number of sessions connected to the server.\nnum_queries\nThe number of queries.\nnum_query_errors_leader_changes\nThe number of the raft leader changes due to query errors.\nnum_query_errors\nThe number of query errors.\nnum_reclaimed_expired_sessions\nThe number of expired sessions actively reclaimed by the server.\nnum_rpc_sent_to_metad_failed\nThe number of failed RPC requests that the Graphd service sent to the Metad service.\nnum_rpc_sent_to_metad\nThe number of RPC requests that the Graphd service sent to the Metad service.\nnum_rpc_sent_to_storaged_failed\nThe number of failed RPC requests that the Graphd service sent to the Storaged service.\nnum_rpc_sent_to_storaged\nThe number of RPC requests that the Graphd service sent to the Storaged service.\nnum_sentences\nThe number of statements received by the Graphd service.\nnum_slow_queries\nThe number of slow queries.\nnum_sort_executors\nThe number of executions for the Sort operator.\noptimizer_latency_us\nThe latency of executing optimizer statements.\nquery_latency_us\nThe latency of queries.\nslow_query_latency_us\nThe latency of slow queries.\nnum_queries_hit_memory_watermark\nThe number of queries reached the memory watermark.\nresp_part_completeness\nThe completeness of the partial success. You need to set accept_partial_success to true in the graph configuration first.\nMeta\nParameter\nDescription\ncommit_log_latency_us\nThe latency of committing logs in Raft.\ncommit_snapshot_latency_us\nThe latency of committing snapshots in Raft.\nheartbeat_latency_us\nThe latency of heartbeats.\nnum_heartbeats\nThe number of heartbeats.\nnum_raft_votes\nThe number of votes in Raft.\ntransfer_leader_latency_us\nThe latency of transferring the raft leader.\nnum_agent_heartbeats\nThe number of heartbeats for the AgentHBProcessor.\nagent_heartbeat_latency_us\nThe latency of the AgentHBProcessor.\nreplicate_log_latency_us\nThe latency of replicating the log record to most nodes by Raft.\nnum_send_snapshot\nThe number of times that Raft sends snapshots to other nodes.\nappend_log_latency_us\nThe latency of replicating the log record to a single node by Raft.\nappend_wal_latency_us\nThe Raft write latency for a single WAL.\nnum_grant_votes\nThe number of times that Raft votes for other nodes.\nnum_start_elect\nThe number of times that Raft starts an election.\nStorage\nParameter\nDescription\nadd_edges_latency_us\nThe latency of adding edges.\nadd_vertices_latency_us\nThe latency of adding vertices.\ncommit_log_latency_us\nThe latency of committing logs in Raft.\ncommit_snapshot_latency_us\nThe latency of committing snapshots in Raft.\ndelete_edges_latency_us\nThe latency of deleting edges.\ndelete_vertices_latency_us\nThe latency of deleting vertices.\nget_neighbors_latency_us\nThe latency of querying neighbor vertices.\nget_dst_by_src_latency_us\nThe latency of querying the destination vertex by the source vertex.\nnum_get_prop\nThe number of executions for the GetPropProcessor.\nnum_get_neighbors_errors\nThe number of execution errors for the GetNeighborsProcessor.\nnum_get_dst_by_src_errors\nThe number of execution errors for the GetDstBySrcProcessor.\nget_prop_latency_us\nThe latency of executions for the GetPropProcessor.\nnum_edges_deleted\nThe number of deleted edges.\nnum_edges_inserted\nThe number of inserted edges.\nnum_raft_votes\nThe number of votes in Raft.\nnum_rpc_sent_to_metad_failed\nThe number of failed RPC requests that the Storage service sent to the Meta service.\nnum_rpc_sent_to_metad\nThe number of RPC requests that the Storaged service sent to the Metad service.\nnum_tags_deleted\nThe number of deleted tags.\nnum_vertices_deleted\nThe number of deleted vertices.\nnum_vertices_inserted\nThe number of inserted vertices.\ntransfer_leader_latency_us\nThe latency of transferring the raft leader.\nlookup_latency_us\nThe latency of executions for the LookupProcessor.\nnum_lookup_errors\nThe number of execution errors for the LookupProcessor.\nnum_scan_vertex\nThe number of executions for the ScanVertexProcessor.\nnum_scan_vertex_errors\nThe number of execution errors for the ScanVertexProcessor.\nupdate_edge_latency_us\nThe latency of executions for the UpdateEdgeProcessor.\nnum_update_vertex\nThe number of executions for the UpdateVertexProcessor.\nnum_update_vertex_errors\nThe number of execution errors for the UpdateVertexProcessor.\nkv_get_latency_us\nThe latency of executions for the Getprocessor.\nkv_put_latency_us\nThe latency of executions for the PutProcessor.\nkv_remove_latency_us\nThe latency of executions for the RemoveProcessor.\nnum_kv_get_errors\nThe number of execution errors for the GetProcessor.\nnum_kv_get\nThe number of executions for the GetProcessor.\nnum_kv_put_errors\nThe number of execution errors for the PutProcessor.\nnum_kv_put\nThe number of executions for the PutProcessor.\nnum_kv_remove_errors\nThe number of execution errors for the RemoveProcessor.\nnum_kv_remove\nThe number of executions for the RemoveProcessor.\nforward_tranx_latency_us\nThe latency of transmission.\nscan_edge_latency_us\nThe latency of executions for the ScanEdgeProcessor.\nnum_scan_edge_errors\nThe number of execution errors for the ScanEdgeProcessor.\nnum_scan_edge\nThe number of executions for the ScanEdgeProcessor.\nscan_vertex_latency_us\nThe latency of executions for the ScanVertexProcessor.\nnum_add_edges\nThe number of times that edges are added.\nnum_add_edges_errors\nThe number of errors when adding edges.\nnum_add_vertices\nThe number of times that vertices are added.\nnum_start_elect\nThe number of times that Raft starts an election.\nnum_add_vertices_errors\nThe number of errors when adding vertices.\nnum_delete_vertices_errors\nThe number of errors when deleting vertices.\nappend_log_latency_us\nThe latency of replicating the log record to a single node by Raft.\nnum_grant_votes\nThe number of times that Raft votes for other nodes.\nreplicate_log_latency_us\nThe latency of replicating the log record to most nodes by Raft.\nnum_delete_tags\nThe number of times that tags are deleted.\nnum_delete_tags_errors\nThe number of errors when deleting tags.\nnum_delete_edges\nThe number of edge deletions.\nnum_delete_edges_errors\nThe number of errors when deleting edges\nnum_send_snapshot\nThe number of times that snapshots are sent.\nupdate_vertex_latency_us\nThe latency of executions for the UpdateVertexProcessor.\nappend_wal_latency_us\nThe Raft write latency for a single WAL.\nnum_update_edge\nThe number of executions for the UpdateEdgeProcessor.\ndelete_tags_latency_us\nThe latency of deleting tags.\nnum_update_edge_errors\nThe number of execution errors for the UpdateEdgeProcessor.\nnum_get_neighbors\nThe number of executions for the GetNeighborsProcessor.\nnum_get_dst_by_src\nThe number of executions for the GetDstBySrcProcessor.\nnum_get_prop_errors\nThe number of execution errors for the GetPropProcessor.\nnum_delete_vertices\nThe number of times that vertices are deleted.\nnum_lookup\nThe number of executions for the LookupProcessor.\nnum_sync_data\nThe number of times the Storage service synchronizes data from the Drainer.\nnum_sync_data_errors\nThe number of errors that occur when the Storage service synchronizes data from the Drainer.\nsync_data_latency_us\nThe latency of the Storage service synchronizing data from the Drainer.\nGraph space\nParameter\nDescription\nnum_active_queries\nThe number of queries currently being executed.\nnum_queries\nThe number of queries.\nnum_sentences\nThe number of statements received by the Graphd service.\noptimizer_latency_us\nThe latency of executing optimizer statements.\nquery_latency_us\nThe latency of queries.\nnum_slow_queries\nThe number of slow queries.\nnum_query_errors\nThe number of query errors.\nnum_query_errors_leader_changes\nThe number of raft leader changes due to query errors.\nnum_killed_queries\nThe number of killed queries.\nnum_aggregate_executors\nThe number of executions for the Aggregation operator.\nnum_sort_executors\nThe number of executions for the Sort operator.\nnum_indexscan_executors\nThe number of executions for index scan operators.\nnum_auth_failed_sessions_bad_username_password\nThe number of sessions where authentication failed due to incorrect username and password.\nnum_auth_failed_sessions\nThe number of sessions in which login authentication failed.\nnum_opened_sessions\nThe number of sessions connected to the server.\nnum_queries_hit_memory_watermark\nThe number of queries reached the memory watermark.\nnum_reclaimed_expired_sessions\nThe number of expired sessions actively reclaimed by the server.\nnum_rpc_sent_to_metad_failed\nThe number of failed RPC requests that the Graphd service sent to the Metad service.\nnum_rpc_sent_to_metad\nThe number of RPC requests that the Graphd service sent to the Metad service.\nnum_rpc_sent_to_storaged_failed\nThe number of failed RPC requests that the Graphd service sent to the Storaged service.\nnum_rpc_sent_to_storaged\nThe number of RPC requests that the Graphd service sent to the Storaged service.\nslow_query_latency_us\nThe latency of slow queries.","url":"https://docs.nebula-graph.io/3.6.0/6.monitor-and-metrics/1.query-performance-metrics/","type":"doc"},{"title":"RocksDB statistics","content":"NebulaGraph uses RocksDB as the underlying storage. This topic describes how to collect and show the RocksDB statistics of NebulaGraph.\nEnable RocksDB\nBy default, the function of RocksDB statistics is disabled. To enable RocksDB statistics, you need to:\nModify the --enable_rocksdb_statistics parameter as true in the nebula-storaged.conf file. The default path of the configuration file is /use/local/nebula/etc.\nRestart the service to make the modification valid.\nGet RocksDB statistics\nUsers can use the built-in HTTP service in the storage service to get the following types of statistics. Results in the JSON format are supported.\nAll RocksDB statistics.\nSpecified RocksDB statistics.\nExamples\nUse the following command to get all RocksDB statistics:\ncurl -L \"http://${storage_ip}:${port}/rocksdb_stats\"\nFor example:\ncurl -L \"http://172.28.2.1:19779/rocksdb_stats\"\nrocksdb.blobdb.blob.file.bytes.read=0\nrocksdb.blobdb.blob.file.bytes.written=0\nrocksdb.blobdb.blob.file.bytes.synced=0\n...\nUse the following command to get specified RocksDB statistics:\ncurl -L \"http://${storage_ip}:${port}/rocksdb_stats?stats=${stats_name}\"\nFor example, use the following command to get the information of rocksdb.bytes.read and rocksdb.block.cache.add.\ncurl -L \"http://172.28.2.1:19779/rocksdb_stats?stats=rocksdb.bytes.read,rocksdb.block.cache.add\"\nrocksdb.block.cache.add=14\nrocksdb.bytes.read=1632\nUse the following command to get specified RocksDB statistics in the JSON format:\ncurl -L \"http://${storage_ip}:${port}/rocksdb_stats?stats=${stats_name}&format=json\"\nFor example, use the following command to get the information of rocksdb.bytes.read and rocksdb.block.cache.add and return the results in the JSON format.\ncurl -L \"http://172.28.2.1:19779/rocksdb_stats?stats=rocksdb.bytes.read,rocksdb.block.cache.add&format=json\"\n \"rocksdb.block.cache.add\": 1\n },\n \"rocksdb.bytes.read\": 160","url":"https://docs.nebula-graph.io/3.6.0/6.monitor-and-metrics/2.rocksdb-statistics/","type":"doc"},{"title":"Authentication","content":"NebulaGraph replies on local authentication authentication to implement access control.\nNebulaGraph creates a session when a client connects to it. The session stores information about the connection, including the user information. If the authentication system is enabled, the session will be mapped to corresponding users.\nLocal authentication\nLocal authentication indicates that usernames and passwords are stored locally on the server, with the passwords encrypted. Users will be authenticated when trying to visit NebulaGraph.\nEnable local authentication\nModify the nebula-graphd.conf file (/usr/local/nebula/etc/ is the default path) to set the following parameters:\n--enable_authorize: Set its value to true to enable authentication.\n--failed_login_attempts: This parameter is optional, and you need to add this parameter manually. Specify the attempts of continuously entering incorrect passwords for a single Graph service. When the number exceeds the limitation, your account will be locked. For multiple Graph services, the allowed attempts are number of services * failed_login_attempts.\n--password_lock_time_in_secs: This parameter is optional, and you need to add this parameter manually. Specify the time how long your account is locked after multiple incorrect password entries are entered. Unit: second.\nRestart the NebulaGraph services. For how to restart, see Manage NebulaGraph services.","url":"https://docs.nebula-graph.io/3.6.0/7.data-security/1.authentication/1.authentication/","type":"doc"},{"title":"User management","content":"User management is an indispensable part of NebulaGraph access control. This topic describes how to manage users and roles.\nAfter enabling authentication, only valid users can connect to NebulaGraph and access the resources according to the user roles.\nCREATE USER\nThe root user with the GOD role can run CREATE USER to create a new user.\nSyntax\nCREATE USER [IF NOT EXISTS] [WITH PASSWORD ''];\nIF NOT EXISTS: Detects if the user name exists. The user will be created only if the user name does not exist.\nuser_name: Sets the name of the user. The maximum length is 16 characters.\npassword: Sets the password of the user. The default password is the empty string (''). The maximum length is 24 characters.\nExample\n CREATE USER user1 WITH PASSWORD 'nebula';\n SHOW USERS;\nGRANT ROLE\nUsers with the GOD role or the ADMIN role can run GRANT ROLE to assign a built-in role in a graph space to a user. For more information about NebulaGraph built-in roles, see Roles and privileges.\nSyntax\nGRANT ROLE ON TO ;\nExample\n GRANT ROLE USER ON basketballplayer TO user1;\nREVOKE ROLE\nUsers with the GOD role or the ADMIN role can run REVOKE ROLE to revoke the built-in role of a user in a graph space. For more information about NebulaGraph built-in roles, see Roles and privileges.\nSyntax\nREVOKE ROLE ON FROM ;\nExample\n REVOKE ROLE USER ON basketballplayer FROM user1;\nDESCRIBE USER\nUsers can run DESCRIBE USER to list the roles for a specified user.\nSyntax\nDESCRIBE USER ;\nDESC USER ;\nExample\n DESCRIBE USER user1;\nSHOW ROLES\nUsers can run SHOW ROLES to list the roles in a graph space.\nSyntax\nSHOW ROLES IN ;\nExample\n SHOW ROLES IN basketballplayer;\nCHANGE PASSWORD\nUsers can run CHANGE PASSWORD to set a new password for a user. The old password is needed when setting a new one.\nSyntax\nCHANGE PASSWORD FROM '' TO '';\nExample\n CHANGE PASSWORD user1 FROM 'nebula' TO 'nebula123';\nALTER USER\nThe root user with the GOD role can run ALTER USER to set a new password. The old password is not needed when altering the user.\nSyntax\nALTER USER WITH PASSWORD '';\n- Example\n ALTER USER user2 WITH PASSWORD 'nebula';\nDROP USER\nThe root user with the GOD role can run DROP USER to remove a user.\nSyntax\nDROP USER [IF EXISTS] ;\nExample\n DROP USER user1;\nSHOW USERS\nThe root user with the GOD role can run SHOW USERS to list all the users.\nSyntax\nSHOW USERS;\nExample\n SHOW USERS;","url":"https://docs.nebula-graph.io/3.6.0/7.data-security/1.authentication/2.management-user/","type":"doc"},{"title":"Roles and privileges","content":"A role is a collection of privileges. You can assign a role to a user for access control.\nBuilt-in roles\nNebulaGraph does not support custom roles, but it has multiple built-in roles:\nGOD\nGOD is the original role with all privileges not limited to graph spaces. It is similar to root in Linux and administrator in Windows.\nWhen the Meta Service is initialized, the one and only GOD role user root is automatically created with the password nebula.\nWhen the --enable_authorize parameter in the nebula-graphd.conf file (the default directory is /usr/local/nebula/etc/) is set to true:\nOne cluster can only have one user with the GOD role. This user can manage all graph spaces in a cluster.\nManual authorization of the God role is not supported. Only the root user with the default God role can be used.\nADMIN\nAn ADMIN role can read and write both the Schema and the data in a specific graph space.\nAn ADMIN role of a graph space can grant DBA, USER, and GUEST roles in the graph space to other users.\nDBA\nA DBA role can read and write both the Schema and the data in a specific graph space.\nA DBA role of a graph space CANNOT grant roles to other users.\nUSER\nA USER role can read and write data in a specific graph space.\nThe Schema information is read-only to the USER roles in a graph space.\nGUEST\nA GUEST role can only read the Schema and the data in a specific graph space.\nRole privileges and allowed nGQL\nThe privileges of roles and the nGQL statements that each role can use are listed as follows.\nPrivilege\nGod\nAdmin\nDBA\nUser\nGuest\nAllowed nGQL\nRead space\nUSE, DESCRIBE SPACE\nRead schema\nDESCRIBE TAG, DESCRIBE EDGE, DESCRIBE TAG INDEX, DESCRIBE EDGE INDEX\nWrite schema\nCREATE TAG, ALTER TAG, CREATE EDGE, ALTER EDGE, DROP TAG, DELETE TAG, DROP EDGE, CREATE TAG INDEX, CREATE EDGE INDEX, DROP TAG INDEX, DROP EDGE INDEX\nWrite user\nCREATE USER, DROP USER, ALTER USER\nWrite role\nGRANT, REVOKE\nRead data\nGO, SET, PIPE, MATCH, ASSIGNMENT, LOOKUP, YIELD, ORDER BY, FETCH VERTICES, Find, FETCH EDGES, FIND PATH, LIMIT, GROUP BY, RETURN\nWrite data\nINSERT VERTEX, UPDATE VERTEX, INSERT EDGE, UPDATE EDGE, DELETE VERTEX, DELETE EDGES, DELETE TAG\nShow operations\nSHOW, CHANGE PASSWORD\nJob\nSUBMIT JOB COMPACT, SUBMIT JOB FLUSH, SUBMIT JOB STATS, STOP JOB, RECOVER JOB, BUILD TAG INDEX, BUILD EDGE INDEX,INGEST, DOWNLOAD\nWrite space\nCREATE SPACE, DROP SPACE, CREATE SNAPSHOT, DROP SNAPSHOT, BALANCE, CONFIG","url":"https://docs.nebula-graph.io/3.6.0/7.data-security/1.authentication/3.role-list/","type":"doc"},{"title":"SSL encryption","content":"NebulaGraph supports SSL encrypted transfers between the Client, Graph Service, Meta Service, and Storage Service, and this topic describes how to set up SSL encryption.\nPrecaution\nEnabling SSL encryption will slightly affect the performance, such as causing operation latency.\nCertificate modes\nTo use SSL encryption, SSL certificates are required. NebulaGraph supports two certificate modes.\nSelf-signed certificate mode\nA certificate that is generated by the server itself and signed by itself. In the self-signed certificate mode, the server needs to generate its own SSL certificate and key, and then use its own private key to sign the certificate. It is suitable for building secure communications for systems and applications within a LAN.\nCA-signed certificate mode\nA certificate granted by a trusted third-party Certificate Authority (CA). In the CA signed certificate mode, the server needs to apply for an SSL certificate from a trusted CA and ensure the authenticity and trustworthiness of the certificate through the auditing and signing of the certificate authority center. It is suitable for public network environment, especially for websites, e-commerce and other occasions that need to protect user information security.\nAuthentication policies\nPolicies for the NebulaGraph community edition.\nScene\nTLS\nExternal device access to Graph\nModify the Graph configuration file to add the following parameters:--enable_graph_ssl = true --ca_path=xxxxxx--cert_path=xxxxxx--key_path=xxxxxx\nGraph access Meta\nIn the Graph/Meta configuration file, add the following parameters:--enable_meta_ssl = true--ca_path=xxxxxx--cert_path=xxxxxx--key_path=xxxxxx\nGraph access StorageMeta access Storage\nIn the Graph/Meta/Storage configuration file, add the following parameters:--enable_storage_ssl = true--ca_path=xxxxxx--cert_path=xxxxxx--key_path=xxxxxx\nGraph access Meta/StorageMeta access Storage\nIn the Graph/Meta/Storage configuration file, add the following parameters:--enable_meta_ssl = true--enable_storage_ssl = true--ca_path=xxxxxx--cert_path=xxxxxx--key_path=xxxxxx\nExternal device access to GraphGraph access Meta/StorageMeta access Storage\nIn the Graph/Meta/Storage configuration file, add the following parameters:--enable_ssl = true--ca_path=xxxxxx--cert_path=xxxxxx--key_path=xxxxxx\nThe parameters are described below.\nParameter\nDefault value\nDescription\ncert_path\nThe path to the SSL public key certificate. This certificate is usually a .pem or .crt file, which is used to prove the identity of the server side, and contains information such as the public key, certificate owner, digital signature, and so on.\nkey_path\nThe path to the SSL key. The SSL key is usually a .key file.\npassword_path\n(Optional) The path to the password file for the SSL key. Some SSL keys are encrypted and require a corresponding password to decrypt. We need to store the password in a separate file and use this parameter to specify the path to the password file.\nca_path\nThe path to the SSL root certificate. The root certificate is a special SSL certificate that is considered the highest level in the SSL trust chain and is used to validate and authorize other SSL certificates.\nenable_ssl\nfalse\nWhether to enable SSL encryption in all services. only.\nenable_graph_ssl\nfalse\nWhether to enable SSL encryption in the Graph service only.\nenable_meta_ssl\nfalse\nWhether to enable SSL encryption in the Meta service only.\nenable_storage_ssl\nfalse\nWhether to enable SSL encryption in the Storage service only.\nExample of TLS\nFor example, using self-signed certificates and TLS for data transfers between the client NebulaGraph Python, the Graph service, the Meta service, and the Storage service. You need to set up all three Graph/Meta/Storage configuration files as follows:\n--enable_ssl=true\n--ca_path=xxxxxx\n--cert_path=xxxxxx\n--key_path=xxxxxx\nWhen the changes are complete, restart these services to make the configuration take effect.\nTo connect to the Graph service using NebulaGraph Python, you need to set up a secure socket and add a trusted CA. For code examples, see nebula-test-run.py.","url":"https://docs.nebula-graph.io/3.6.0/7.data-security/4.ssl/","type":"doc"},{"title":"What is Backup & Restore","content":"Backup & Restore (BR for short) is a Command-Line Interface (CLI) tool to back up data of graph spaces of NebulaGraph and to restore data from the backup files.\nFeatures\nThe BR has the following features. It supports:\nBacking up and restoring data in a one-click operation.\nRestoring data in the following backup file types:\nLocal Disk (SSD or HDD). It is recommend to use local disk in test environment only.\nAmazon S3 compatible interface, such as Alibaba Cloud OSS, MinIO,Ceph RGW, etc.\nBacking up and restoring the entire NebulaGraph cluster.\nBacking up data of specified graph spaces (experimental).\nLimitations\nSupports NebulaGraph v3.x only.\nSupports full backup, but not incremental backup.\nCurrently, NebulaGraph Listener and full-text indexes do not support backup.\nIf you back up data to the local disk, the backup files will be saved in the local path of each server. You can also mount the NFS on your host to restore the backup data to a different host.\nRestoration requires that the number of the storage servers in the original cluster is the same as that of the storage servers in the target cluster and storage server IPs must be the same. Restoring the specified space will clear all the remaining spaces in the cluster.\nDuring the backup process, both DDL and DML statements in any specified graph spaces are blocked. We recommend that you do the operation within the low peak period of the business, for example, from 2:00 AM to 5:00 AM.\nDuring the restoration process, there is a time when NebulaGraph stops running.\nUsing BR in a container-based NebulaGraph cluster is not supported.\nHow to use BR\nTo use the BR, follow these steps:\nInstall BR.\nUse BR to back up data.\nUse BR to restore data from backup files.","url":"https://docs.nebula-graph.io/3.6.0/backup-and-restore/nebula-br/1.what-is-br/","type":"doc"},{"title":"Install BR","content":"This topic introduces the installation of BR in bare-metal deployment scenarios. \nNotes\nTo use the BR (Community Edition) tool, you need to install the NebulaGraph Agent service, which is taken as a daemon for each machine in the cluster that starts and stops the NebulaGraph service, and uploads and downloads backup files. The BR (Community Edition) tool and the Agent plug-in are installed as described below.\nVersion compatibility\nNebulaGraph\nBR\nAgent\n3.5.x\n3.5.0\n0.2.0 ~ 3.4.0\n3.3.0 ~ 3.4.1\n3.3.0\n0.2.0 ~ 3.4.0\n3.0.x ~ 3.2.x\n0.6.1\n0.1.0 ~ 0.2.0\nInstall BR with a binary file\nInstall BR.\nwget https://github.com/vesoft-inc/nebula-br/releases/download/v3.5.0/br-3.5.0-linux-amd64\nChange the binary file name to br.\nsudo mv br-3.5.0-linux-amd64 br\nGrand execute permission to BR.\nsudo chmod +x br\nRun ./br version to check BR version.\n[nebula-br]$ ./br version\nNebula Backup And Restore Utility Tool,V-3.5.0\nInstall BR with the source code\nBefore compiling the BR, do a check of these:\nGo 1.14.x or a later version is installed.\nmake is installed.\nTo compile the BR, follow these steps:\nClone the nebula-br repository to your machine.\ngit clone https://github.com/vesoft-inc/nebula-br.git\nChange to the br directory.\ncd nebula-br\nCompile the BR.\nmake\nUsers can enter bin/br version on the command line. If the following results are returned, the BR is compiled successfully.\n[nebula-br]$ bin/br version\nNebulaGraph Backup And Restore Utility Tool,V-3.5.0\nInstall Agent\nNebulaGraph Agent is installed as a binary file in each machine and serves the BR tool with the RPC protocol.\nIn each machine, follow these steps:\nInstall Agent.\nwget https://github.com/vesoft-inc/nebula-agent/releases/download/v3.6.1/agent-3.6.1-linux-amd64\nRename the Agent file to agent.\nsudo mv agent-3.6.1-linux-amd64 agent\nAdd execute permission to Agent. \nsudo chmod +x agent\nStart Agent.\nsudo nohup ./agent --agent=\":8888\" --meta=\":9559\" --ratelimit= > nebula_agent.log 2>&1 &\n--agent: The IP address and port number of Agent.\n--meta: The IP address and access port of any Meta service in the cluster.\n--ratelimit: (Optional) Limits the speed of file uploads and downloads to prevent bandwidth from being filled up and making other services unavailable. Unit: Bytes.\nFor example: \nsudo nohup ./agent --agent=\"192.168.8.129:8888\" --meta=\"192.168.8.129:9559\" --ratelimit=1048576 > nebula_agent.log 2>&1 &\nLog into NebulaGraph and then run the following command to view the status of Agent.\n SHOW HOSTS AGENT;\n+-----------------+------+----------+---------+--------------+---------+ \nFAQ\nThe error `E_LIST_CLUSTER_NO_AGENT_FAILURE\nIf you encounter E_LIST_CLUSTER_NO_AGENT_FAILURE error, it may be due to the Agent service is not started or the Agent service is not registered to Meta service. First, execute SHOW HOSTS AGENT to check the status of the Agent service on all nodes in the cluster, when the status shows OFFLINE, it means the registration of Agent failed, then check whether the value of the --meta option in the command to start the Agent service is correct.","url":"https://docs.nebula-graph.io/3.6.0/backup-and-restore/nebula-br/2.compile-br/","type":"doc"},{"title":"Use BR to back up data","content":"After the BR is installed, you can back up data of the entire graph space. This topic introduces how to use the BR to back up data.\nPrerequisites\nTo back up data with the BR, do a check of these:\nInstall BR and Agent and run Agent on each host in the cluster.\nThe NebulaGraph services are running.\nIf you store the backup files locally, create a directory with the same absolute path on the meta servers, the storage servers, and the BR machine for the backup files and get the absolute path. Make sure the account has write privileges for this directory.\nProcedure\nIn the BR installation directory (the default path of the compiled BR is ./bin/br), run the following command to perform a full backup for the entire cluster.\n$ ./br backup full --meta --storage \nFor example: \nRun the following command to perform a full backup for the entire cluster whose meta service address is 192.168.8.129:9559, and save the backup file to /home/nebula/backup/.\n$ ./br backup full --meta \"192.168.8.129:9559\" --storage \"local:///home/nebula/backup/\"\nRun the following command to perform a full backup for the entire cluster whose meta service address is 192.168.8.129:9559, and save the backup file to backup in the br-test bucket of the object storage service compatible with S3 protocol.\n$ ./br backup full --meta \"192.168.8.129:9559\" --s3.endpoint \"http://192.168.8.129:9000\" --storage=\"s3://br-test/backup/\" --s3.access_key=minioadmin --s3.secret_key=minioadmin --s3.region=default\nThe parameters are as follows.\nParameter\nData type\nRequired\nDefault value\nDescription\n-h,-help\nNo\nNone\nChecks help for restoration.\n--debug\nNo\nNone\nChecks for more log information.\n--log\nstring\nNo\n\"br.log\"\nSpecifies detailed log path for restoration and backup.\n--meta\nstring\nYes\nNone\nThe IP address and port of the meta service.\n--space\nstring\nYes\nNone\n(Experimental feature) Specifies the names of the spaces to be backed up. All spaces will be backed up if not specified. Multiple spaces can be specified, and format is --spaces nba_01 --spaces nba_02.\n--storage\nstring\nYes\nNone\nThe target storage URL of BR backup data. The format is: \\://\\. Schema: Optional values are local and s3. When selecting s3, you need to fill in s3.access_key, s3.endpoint, s3.region, and s3.secret_key.PATH: The path of the storage location.\n--s3.access_key\nstring\nNo\nNone\nSets AccessKey ID.\n--s3.endpoint\nstring\nNo\nNone\nSets the S3 endpoint URL, please specify the HTTP or HTTPS scheme explicitly.\n--s3.region\nstring\nNo\nNone\nSets the region or location to upload or download the backup.\n--s3.secret_key\nstring\nNo\nNone\nSets SecretKey for AccessKey ID.\nNext to do\nAfter the backup files are generated, you can use the BR to restore them for NebulaGraph. For more information, see Use BR to restore data.","url":"https://docs.nebula-graph.io/3.6.0/backup-and-restore/nebula-br/3.br-backup-data/","type":"doc"},{"title":"Use BR to restore data","content":"If you use the BR to back up data, you can use it to restore the data to NebulaGraph. This topic introduces how to use the BR to restore data from backup files.\nPrerequisites\nInstall BR and Agent and run Agent on each host in the cluster. \nNo application is connected to the target NebulaGraph cluster.\nMake sure that the target and the source NebulaGraph clusters have the same topology, which means that they have exactly the same number of hosts. The number of data folders for each host is consistently distributed.\nProcedures\nIn the BR installation directory (the default path of the compiled BR is ./br), run the following command to perform a full backup for the entire cluster.\nUsers can use the following command to list the existing backup information:\n$ ./br show --storage \nFor example, run the following command to list the backup information in the local /home/nebula/backup path.\n$ ./br show --storage \"local:///home/nebula/backup\"\nOr, you can run the following command to list the backup information stored in S3 URL s3://192.168.8.129:9000/br-test/backup.\n$ ./br show --s3.endpoint \"http://192.168.8.129:9000\" --storage=\"s3://br-test/backup/\" --s3.access_key=minioadmin --s3.secret_key=minioadmin --s3.region=default\nParameter\nData type\nRequired\nDefault value\nDescription\n-h,-help\nNo\nNone\nChecks help for restoration.\n-debug\nNo\nNone\nChecks for more log information.\n-log\nstring\nNo\n\"br.log\"\nSpecifies detailed log path for restoration and backup.\n--storage\nstring\nYes\nNone\nThe target storage URL of BR backup data. The format is: ://. Schema: Optional values are local and s3. When selecting s3, you need to fill in s3.access_key, s3.endpoint, s3.region, and s3.secret_key.PATH: The path of the storage location.\n--s3.access_key\nstring\nNo\nNone\nSets AccessKey ID.\n--s3.endpoint\nstring\nNo\nNone\nSets the S3 endpoint URL, please specify the HTTP or HTTPS scheme explicitly.\n--s3.region\nstring\nNo\nNone\nSets the region or location to upload or download the backup.\n--s3.secret_key\nstring\nNo\nNone\nSets SecretKey for AccessKey ID.\nRun the following command to restore data.\n$ ./br restore full --meta --storage --name \nFor example, run the following command to upload the backup files from the local /home/nebula/backup/ to the cluster where the meta service's address is 192.168.8.129:9559.\n$ ./br restore full --meta \"192.168.8.129:9559\" --storage \"local:///home/nebula/backup/\" --name BACKUP_2021_12_08_18_38_08\nOr, you can run the following command to upload the backup files from the S3 URL s3://192.168.8.129:9000/br-test/backup.\n$ ./br restore full --meta \"192.168.8.129:9559\" --s3.endpoint \"http://192.168.8.129:9000\" --storage=\"s3://br-test/backup/\" --s3.access_key=minioadmin --s3.secret_key=minioadmin --s3.region=\"default\" --name BACKUP_2021_12_08_18_38_08\nIf the following information is returned, the data is restored successfully.\nRestore succeed.\nThe parameters are as follows.\nParameter\nData type\nRequired\nDefault value\nDescription\n-h,-help\nNo\nNone\nChecks help for restoration.\n-debug\nNo\nNone\nChecks for more log information.\n-log\nstring\nNo\n\"br.log\"\nSpecifies detailed log path for restoration and backup.\n-meta\nstring\nYes\nNone\nThe IP address and port of the meta service.\n-name\nstring\nYes\nNone\nThe name of backup.\n--storage\nstring\nYes\nNone\nThe target storage URL of BR backup data. The format is: \\://\\. Schema: Optional values are local and s3. When selecting s3, you need to fill in s3.access_key, s3.endpoint, s3.region, and s3.secret_key.PATH: The path of the storage location.\n--s3.access_key\nstring\nNo\nNone\nSets AccessKey ID.\n--s3.endpoint\nstring\nNo\nNone\nSets the S3 endpoint URL, please specify the HTTP or HTTPS scheme explicitly.\n--s3.region\nstring\nNo\nNone\nSets the region or location to upload or download the backup.\n--s3.secret_key\nstring\nNo\nNone\nSets SecretKey for AccessKey ID.\nRun the following command to clean up temporary files if any error occurred during backup. It will clean the files in cluster and external storage. You could also use it to clean up old backups files in external storage.\n$ ./br cleanup --meta --storage --name \nThe parameters are as follows.\nParameter\nData type\nRequired\nDefault value\nDescription\n-h,-help\nNo\nNone\nChecks help for restoration.\n-debug\nNo\nNone\nChecks for more log information.\n-log\nstring\nNo\n\"br.log\"\nSpecifies detailed log path for restoration and backup.\n-meta\nstring\nYes\nNone\nThe IP address and port of the meta service.\n-name\nstring\nYes\nNone\nThe name of backup.\n--storage\nstring\nYes\nNone\nThe target storage URL of BR backup data. The format is: \\://\\. Schema: Optional values are local and s3. When selecting s3, you need to fill in s3.access_key, s3.endpoint, s3.region, and s3.secret_key.PATH: The path of the storage location.\n--s3.access_key\nstring\nNo\nNone\nSets AccessKey ID.\n--s3.endpoint\nstring\nNo\nNone\nSets the S3 endpoint URL, please specify the HTTP or HTTPS scheme explicitly.\n--s3.region\nstring\nNo\nNone\nSets the region or location to upload or download the backup.\n--s3.secret_key\nstring\nNo\nNone\nSets SecretKey for AccessKey ID.","url":"https://docs.nebula-graph.io/3.6.0/backup-and-restore/nebula-br/4.br-restore-data/","type":"doc"},{"title":"Backup and restore data with snapshots","content":"NebulaGraph supports using snapshots to back up and restore data. When data loss or misoperation occurs, the data will be restored through the snapshot.\nPrerequisites\nNebulaGraph authentication is disabled by default. In this case, all users can use the snapshot feature.\nIf authentication is enabled, only the GOD role user can use the snapshot feature. For more information about roles, see Roles and privileges.\nPrecautions\nTo prevent data loss, create a snapshot as soon as the system structure changes, for example, after operations such as ADD HOST, DROP HOST, CREATE SPACE, DROP SPACE, and BALANCE are performed.\nNebulaGraph cannot automatically delete the invalid files created by a failed snapshot task. You have to manually delete them by using DROP SNAPSHOT.\nCustomizing the storage path for snapshots is not supported for now.\nCreate snapshots\nRun CREATE SNAPSHOT to create a snapshot for all the graph spaces based on the current time for NebulaGraph. Creating a snapshot for a specific graph space is not supported yet.\n CREATE SNAPSHOT;\nView snapshots\nTo view all existing snapshots, run SHOW SNAPSHOTS.\n SHOW SNAPSHOTS;\nThe parameters in the return information are described as follows.\nParameter\nDescription\nName\nThe name of the snapshot directory. The prefix SNAPSHOT indicates that the file is a snapshot file, and the suffix indicates the time the snapshot was created (UTC).\nStatus\nThe status of the snapshot. VALID indicates that the creation succeeded, while INVALID indicates that it failed.\nHosts\nThe IPs (or hostnames) and ports of all Storage servers at the time the snapshot was created.\nSnapshot path\nSnapshots are stored in the path specified by the data_path parameter in the Meta and Storage configuration files. When a snapshot is created, the checkpoints directory is checked in the datastore path of the leader Meta service and all Storage services for the existence, and if it is not there, it is automatically created. The newly created snapshot is stored as a subdirectory within the checkpoints directory. For example, SNAPSHOT_2021_03_09_08_43_12. The suffix 2021_03_09_08_43_12 is generated automatically based on the creation time (UTC).\nTo fast locate the path where the snapshots are stored, you can use the Linux command find in the datastore path. For example:\n$ cd /usr/local/nebula-graph-ent-3.6.0/data\n$ find |grep 'SNAPSHOT_2021_03_09_08_43_12'\n./data/meta2/nebula/0/checkpoints/SNAPSHOT_2021_03_09_08_43_12\n./data/meta2/nebula/0/checkpoints/SNAPSHOT_2021_03_09_08_43_12/data\n./data/meta2/nebula/0/checkpoints/SNAPSHOT_2021_03_09_08_43_12/data/000081.sst\n...\nDelete snapshots\nTo delete a snapshot with the given name, run DROP SNAPSHOT.\nDROP SNAPSHOT ;\nExample:\n DROP SNAPSHOT SNAPSHOT_2021_03_09_08_43_12;\n SHOW SNAPSHOTS;\nRestore data with snapshots\nCurrently, there is no command to restore data with snapshots. You need to manually copy the snapshot file to the corresponding folder, or you can make it by using a shell script. The logic implements as follows:\nAfter the snapshot is created, the checkpoints directory is generated in the installation directory of the leader Meta service and all Storage services, and saves the created snapshot. Taking this topic as an example, when there are two graph spaces, the snapshots created are saved in /usr/local/nebula/data/meta/nebula/0/checkpoints, /usr/local/nebula/data/storage/ nebula/3/checkpoints and /usr/local/nebula/data/storage/nebula/4/checkpoints.\n$ ls /usr/local/nebula/data/meta/nebula/0/checkpoints/\nSNAPSHOT_2021_03_09_09_10_52\n$ ls /usr/local/nebula/data/storage/nebula/3/checkpoints/\nSNAPSHOT_2021_03_09_09_10_52\n$ ls /usr/local/nebula/data/storage/nebula/4/checkpoints/\nSNAPSHOT_2021_03_09_09_10_52\nTo restore the lost data through snapshots, you can take a snapshot at an appropriate time, copy the folders data and wal in the corresponding snapshot directory to its parent directory (at the same level with checkpoints) to overwrite the previous data and wal, and then restart the cluster.","url":"https://docs.nebula-graph.io/3.6.0/backup-and-restore/3.manage-snapshot/","type":"doc"},{"title":"BALANCE syntax","content":"We can submit tasks to load balance Storage services in NebulaGraph. For more information about storage load balancing and examples, see Storage load balance.\nThe syntax for load balance is described as follows.\nSyntax\nDescription\nSUBMIT JOB BALANCE LEADER\nStarts a job to balance the distribution of all the storage leaders in all graph spaces. It returns the job ID.\nFor details about how to view, stop, and restart a job, see Job manager and the JOB statements.","url":"https://docs.nebula-graph.io/3.6.0/synchronization-and-migration/2.balance-syntax/","type":"doc"},{"title":"Import tools","content":"There are many ways to write NebulaGraph 3.6.0:\nImport with the command -f: This method imports a small number of prepared nGQL files, which is suitable to prepare for a small amount of manual test data.\nImport with Studio: This method uses a browser to import multiple csv files of this machine. A single file cannot exceed 100 MB, and its format is limited.\nImport with Importer: This method imports multiple csv files on a single machine with unlimited size and flexible format.\nImport with Exchange: This method imports from various distribution sources, such as Neo4j, Hive, MySQL, etc., which requires a Spark cluster.\nImport with Spark-connector/Flink-connector: This method has corresponding components (Spark/Flink) and writes a small amount of code.\nImport with C++/GO/Java/Python SDK: This method imports in the way of writing programs, which requires certain programming and tuning skills.\nThe following figure shows the positions of these ways:","url":"https://docs.nebula-graph.io/3.6.0/import-export/write-tools/","type":"doc"},{"title":"NebulaGraph Importer","content":"NebulaGraph Importer (Importer) is a standalone tool for importing data from CSV files into NebulaGraph. Importer can read and batch import CSV file data from multiple data sources, and also supports batch update and delete operations.\nFeatures\nSupport multiple data sources, including local, S3, OSS, HDFS, FTP, SFTP, and GCS.\nSupport importing data from CSV format files. A single file can contain multiple tags, multiple edge types or a mix of both.\nSupport filtering the data from source.\nSupport batch operation, including insert, update, delete.\nSupport connecting to multiple Graph services simultaneously for importing and dynamic load balancing.\nSupport reconnect or retry after failure.\nSupport displaying statistics in multiple dimensions, including import time, import percentage, etc. Support for printing statistics in Console or logs.\nSupport SSL.\nAdvantage\nLightweight and fast: no complex environment can be used, fast data import.\nFlexible filtering: You can flexibly filter CSV data through configuration files.\nVersion compatibility\nThe version correspondence between NebulaGraph and NebulaGraph Importer is as follows.\nNebulaGraph version\nNebulaGraph Importer version\n3.x.x\n3.x.x, 4.x.x\n2.x.x\n2.x.x, 3.x.x\nRelease note\nRelease\nPrerequisites\nBefore using NebulaGraph Importer, make sure:\nNebulaGraph service has been deployed. The deployment method is as follows:\nDeploy NebulaGraph with Docker Compose\nInstall NebulaGraph with RPM or DEB package\nInstall NebulaGraph by compiling the source code\nSchema is created in NebulaGraph, including space, Tag and Edge type, or set by parameter manager.hooks.before.statements.\nSteps\nPrepare the CSV file to be imported and configure the YAML file to use the tool to batch write data into NebulaGraph.\nDownload binary package and run\nDownload the executable binary package.\nUnder the directory where the binary file is located, run the following command to start importing data.\n./ --config \nSource code compile and run\nCompiling the source code requires deploying a Golang environment. For details, see Build Go environment.\nClone repository.\ngit clone -b release-4.1 https://github.com/vesoft-inc/nebula-importer.git\nAccess the directory nebula-importer.\ncd nebula-importer\nCompile the source code.\nmake build\nStart the service.\n./bin/nebula-importer --config \nRun in Docker mode\nInstead of installing the Go locale locally, you can use Docker to pull the image of the NebulaGraph Importer and mount the local configuration file and CSV data file into the container. The command is as follows:\ndocker pull vesoft/nebula-importer:\ndocker run --rm -ti \\\n --network=host \\\n -v : \\\n -v : \\\n vesoft/nebula-importer: \\\n --config \n: The absolute path to the YAML configuration file.\n: The absolute path to the CSV data file. If the file is not local, ignore this parameter.\n: NebulaGraph 3.x Please fill in 'v3'.\nExample:\ndocker pull vesoft/nebula-importer:v4\ndocker run --rm -ti \\\n --network=host \\\n -v /home/user/config.yaml:/home/user/config.yaml \\\n -v /home/user/data:/home/user/data \\\n vesoft/nebula-importer:v4 \\\n --config /home/user/config.yaml\nConfiguration File Description\nVarious example configuration files are available within the Github of the NebulaGraph Importer. The configuration files are used to describe information about the files to be imported, NebulaGraph server information, etc. The following section describes the fields within the configuration file in categories.\nClient configuration\nClient configuration stores the configuration associated with the client's connection to the NebulaGraph.\nThe example configuration is as follows:\nclient:\n version: v3\n address: \"192.168.1.100:9669,192.168.1.101:9669\"\n user: root\n password: nebula\n ssl:\n enable: true\n certPath: \"/home/xxx/cert/importer.crt\"\n keyPath: \"/home/xxx/cert/importer.key\"\n caPath: \"/home/xxx/cert/root.crt\"\n insecureSkipVerify: false\n concurrencyPerAddress: 10\n reconnectInitialInterval: 1s\n retry: 3\n retryInitialInterval: 1s\nParameter\nDefault value\nRequired\nDescription\nclient.version\nv3\nYes\nSpecifies the major version of the NebulaGraph. Currently only v3 is supported.\nclient.address\n\"127.0.0.1:9669\"\nYes\nSpecifies the address of the NebulaGraph. Multiple addresses are separated by commas.\nclient.user\nroot\nNo\nNebulaGraph user name.\nclient.password\nnebula\nNo\nThe password for the NebulaGraph user name.\nclient.ssl.enable\nfalse\nNo\nSpecifies whether to enable SSL authentication.\nclient.ssl.certPath\nNo\nSpecifies the storage path for the SSL public key certificate.This parameter is required when SSL authentication is enabled.\nclient.ssl.keyPath\nNo\nS pecifies the storage path for the SSL key.This parameter is required when SSL authentication is enabled.\nclient.ssl.caPath\nNo\nSpecifies the storage path for the CA root certificate.This parameter is required when SSL authentication is enabled.\nclient.ssl.insecureSkipVerify\nfalse\nNo\nSpecifies whether the client skips verifying the server's certificate chain and hostname. If set to true, any certificate chain and hostname provided by the server is accepted.\nclient.concurrencyPerAddress\n10\nNo\nThe number of concurrent client connections for a single graph service.\nclient.retryInitialInterval\n1s\nNo\nReconnect interval time.\nclient.retry\nNo\nThe number of retries for failed execution of the nGQL statement.\nclient.retryInitialInterval\n1s\nNo\nRetry interval time.\nManager configuration\nManager configuration is a human-controlled configuration after connecting to the database.\nThe example configuration is as follows:\nmanager:\n spaceName: basic_string_examples\n batch: 128\n readerConcurrency: 50\n importerConcurrency: 512\n statsInterval: 10s\n hooks:\n before:\n - statements:\n - UPDATE CONFIGS storage:wal_ttl=3600;\n - UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = true };\n - statements:\n - |\n DROP SPACE IF EXISTS basic_string_examples;\n CREATE SPACE IF NOT EXISTS basic_string_examples(partition_num=5, replica_factor=1, vid_type=int);\n USE basic_string_examples;\n wait: 10s\n after:\n - statements:\n - |\n UPDATE CONFIGS storage:wal_ttl=86400;\n UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = false };\nParameter\nDefault value\nRequired\nDescription\nmanager.spaceName\nYes\nSpecifies the NebulaGraph space to import the data into. Do not support importing multiple map spaces at the same time.\nmanager.batch\n128\nNo\nThe batch size for executing statements (global configuration).\nSetting the batch size individually for a data source can using the parameter sources.batch below.\nmanager.readerConcurrency\n50\nNo\nThe number of concurrent reads of the data source by the reader.\nmanager.importerConcurrency\n512\nNo\nThe number of concurrent nGQL statements generated to be executed, and then will call the client to execute these nGQL statements.\nmanager.statsInterval\n10s\nNo\nThe time interval for printing statistical information\nmanager.hooks.before.[].statements\nNo\nThe command to execute in the graph space before importing.\nmanager.hooks.before.[].wait\nNo\nThe wait time after statements are executed.\nmanager.hooks.after.[].statements\nNo\nThe commands to execute in the graph space after importing.\nmanager.hooks.after.[].wait\nNo\nThe wait time after statements are executed.\nLog configuration\nLog configuration is the logging-related configuration.\nThe example configuration is as follows:\nlog:\n level: INFO\n console: true\n files:\n - logs/nebula-importer.log\nParameter\nDefault value\nRequired\nDescription\nlog.level\nINFO\nNo\nSpecifies the log level. Optional values are DEBUG, INFO, WARN, ERROR, PANIC, FATAL.\nlog.console\ntrue\nNo\nWhether to print the logs to console synchronously when storing logs.\nlog.files\nNo\nThe log file path. The log directory must exist.\nSource configuration\nThe Source configuration requires the configuration of data source information, data processing methods, and Schema mapping.\nThe example configuration is as follows:\nsources:\n - path: ./person.csv # Required. Specifies the path where the data files are stored. If a relative path is used, the path and current configuration file directory are spliced. Wildcard filename is also supported, for example: ./follower-*.csv, please make sure that all matching files with the same schema.\n# - s3: # AWS S3\n# endpoint: endpoint # Optional. The endpoint of S3 service, can be omitted if using AWS S3.\n# region: us-east-1 # Required. The region of S3 service.\n# bucket: gdelt-open-data # Required. The bucket of file in S3 service.\n# key: events/20190918.export.csv # Required. The object key of file in S3 service.\n# accessKeyID: \"\" # Optional. The access key of S3 service. If it is public data, no need to configure.\n# accessKeySecret: \"\" # Optional. The secret key of S3 service. If it is public data, no need to configure.\n# - oss:\n# endpoint: https://oss-cn-hangzhou.aliyuncs.com # Required. The endpoint of OSS service.\n# bucket: bucketName # Required. The bucket of file in OSS service.\n# key: objectKey # Required. The object key of file in OSS service.\n# accessKeyID: accessKey # Required. The access key of OSS service.\n# accessKeySecret: secretKey # Required. The secret key of OSS service.\n# - ftp:\n# host: 192.168.0.10 # Required. The host of FTP service.\n# port: 21 # Required. The port of FTP service.\n# user: user # Required. The user of FTP service.\n# password: password # Required. The password of FTP service.\n# path: \"/events/20190918.export.csv\" # Required. The path of file in the FTP service.\n# - sftp:\n# host: 192.168.0.10 # Required. The host of SFTP service.\n# port: 22 # Required. The port of SFTP service.\n# user: user # Required. The user of SFTP service.\n# password: password # Optional. The password of SFTP service.\n# keyFile: keyFile # Optional. The ssh key file path of SFTP service.\n# keyData: keyData $ Optional. The ssh key file content of SFTP service.\n# passphrase: passphrase # Optional. The ssh key passphrase of SFTP service.\n# path: \"/events/20190918.export.csv\" # Required. The path of file in the SFTP service.\n# - hdfs:\n# address: \"127.0.0.1:8020\" # Required. The address of HDFS service.\n# user: \"hdfs\" # Optional. The user of HDFS service.\n# servicePrincipalName: # Optional. The name of the Kerberos service instance for the HDFS service when Kerberos authentication is enabled.\n# krb5ConfigFile: # Optional. The path to the Kerberos configuration file for the HDFS service when Kerberos authentication is enabled. Defaults to `/etc/krb5.conf`.\n# ccacheFile: # Optional. The path to the Kerberos ccache file for the HDFS service when Kerberos authentication is enabled.\n# keyTabFile: # Optional. The path to the Kerberos keytab file for the HDFS service when Kerberos authentication is enabled.\n# password: # Optional. The Kerberos password for the HDFS service when Kerberos authentication is enabled.\n# dataTransferProtection: # Optional. The type of transport encryption when Kerberos authentication is enabled. Optional values are `authentication`, `integrity`, `privacy`.\n# disablePAFXFAST: false # Optional. Whether to disable the use of PA_FX_FAST for clients.\n# path: \"/events/20190918.export.csv\" # Required. The path to the file in the HDFS service. Wildcard filenames are also supported, e.g. `/events/*.export.csv`, make sure all matching files have the same schema.\n# - gcs: # Google Cloud Storage\n# bucket: chicago-crime-sample # Required. The name of the bucket in the GCS service.\n# key: stats/000000000000.csv # Required. The path to the file in the GCS service.\n# withoutAuthentication: false # Optional. Whether to anonymize access. Defaults to false, which means access with credentials.\n# # When using credentials access, one of the credentialsFile and credentialsJSON parameters is sufficient.\n# credentialsFile: \"/path/to/your/credentials/file\" # Optional. The path to the credentials file for the GCS service.\n# credentialsJSON: '{ # Optional. The JSON content of the credentials for the GCS service.\n# \"type\": \"service_account\",\n# \"project_id\": \"your-project-id\",\n# \"private_key_id\": \"key-id\",\n# \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nxxxxx\\n-----END PRIVATE KEY-----\\n\",\n# \"client_email\": \"your-client@your-project-id.iam.gserviceaccount.com\",\n# \"client_id\": \"client-id\",\n# \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n# \"token_uri\": \"https://oauth2.googleapis.com/token\",\n# \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n# \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/your-client%40your-project-id.iam.gserviceaccount.com\",\n# \"universe_domain\": \"googleapis.com\"\n# }'\n batch: 256\n csv:\n delimiter: \"|\"\n withHeader: false\n lazyQuotes: false\n tags:\n - name: Person\n# mode: INSERT\n# filter: \n# expr: Record[1] == \"XXX\" \n id:\n type: \"STRING\"\n function: \"hash\"\n# index: 0 \n concatItems:\n - person_\n - 0\n - _id\n props:\n - name: \"firstName\"\n type: \"STRING\"\n index: 1\n - name: \"lastName\"\n type: \"STRING\"\n index: 2\n - name: \"gender\"\n type: \"STRING\"\n index: 3\n nullable: true\n defaultValue: female\n - name: \"birthday\"\n type: \"DATE\"\n index: 4\n nullable: true\n nullValue: _NULL_\n - name: \"creationDate\"\n type: \"DATETIME\"\n index: 5\n - name: \"locationIP\"\n type: \"STRING\"\n index: 6\n - name: \"browserUsed\"\n type: \"STRING\"\n index: 7\n - path: ./knows.csv\n batch: 256\n edges:\n - name: KNOWS # person_knows_person\n# mode: INSERT\n# filter: \n# expr: Record[1] == \"XXX\"\n src:\n id:\n type: \"STRING\"\n concatItems:\n - person_\n - 0\n - _id\n dst:\n id:\n type: \"STRING\"\n concatItems:\n - person_\n - 1\n - _id\n props:\n - name: \"creationDate\"\n type: \"DATETIME\"\n index: 2\n nullable: true\n nullValue: _NULL_\n defaultValue: 0000-00-00T00:00:00\nThe configuration mainly includes the following parts:\nSpecify the data source information.\nSpecifies the batch size for executing statements.\nSpecifies the CSV file format information.\nSpecifies the schema mapping for Tag.\nSpecifies the schema mapping for Edge type.\nParameter\nDefault value\nRequired\nDescription\nsources.pathsources.s3sources.osssources.ftpsources.sftpsources.hdfs\nNo\nSpecify data source information, such as local file, HDFS, and S3. Only one source can be configured for the source. Configure multiple sources in multiple source.See the comments in the example for configuration items for different data sources.\nsources.batch\n256\nNo\nThe batch size for executing statements when importing this data source. The priority is higher than manager.batch.\nsources.csv.delimiter\nNo\nSpecifies the delimiter for the CSV file. Only 1-character string separators are supported. Special characters like tabs (\\t) and hexadecimal values (e.g., 0x03 or Ctrl+C) must be properly escaped and enclosed in double quotes, such as \"\\t\" for tabs and \"\\x03\" or \"\\u0003\" for hexadecimal values, instead of using single quotes. For details on escaping special characters in yaml format, see Escaped Characters.\nsources.csv.withHeader\nfalse\nNo\nWhether to ignore the first record in the CSV file.\nsources.csv.lazyQuotes\nfalse\nNo\nWhether to allow lazy quotes. If lazyQuotes is true, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field.\nsources.tags.name\nYes\nThe tag name.\nsources.tags.mode\nINSERT\nNo\nBatch operation types, including insert, update and delete. Optional values are INSERT, UPDATE and DELETE.\nsources.tags.filter.expr\nNo\nFilter the data and only import if the filter conditions are met. Supported comparison characters are ==, ! =, <, >, <= and >=. Logical operators supported are not (!) , and (&&) and or (||). For example (Record[0] == \"Mahinda\" or Record[0] == \"Michael\") and Record[3] == \"male\".\nsources.tags.id.type\nSTRING\nNo\nThe type of the VID.\nsources.tags.id.function\nNo\nFunctions to generate the VID. Currently, only function hash are supported.\nsources.tags.id.index\nNo\nThe column number corresponding to the VID in the data file. If sources.tags.id.concatItems is not configured, this parameter must be configured.\nsources.tags.id.concatItems\nNo\nUsed to concatenate two or more arrays, the concatenated items can be string, int or mixed. string stands for a constant, int for an index column. If this parameter is set, the sources.tags.id.index parameter will not take effect.\nsources.tags.ignoreExistedIndex\ntrue\nNo\nWhether to enable IGNORE_EXISTED_INDEX, that is, do not update index after insertion vertex.\nsources.tags.props.name\nYes\nThe tag property name, which must match the Tag property in the database.\nsources.tags.props.type\nSTRING\nNo\nProperty data type, supporting BOOL, INT, FLOAT, DOUBLE, STRING, TIME, TIMESTAMP, DATE, DATETIME, GEOGRAPHY, GEOGRAPHY(POINT), GEOGRAPHY(LINESTRING) and GEOGRAPHY(POLYGON).\nsources.tags.props.index\nYes\nThe property corresponds to the column number in the data file.\nsources.tags.props.nullable\nfalse\nNo\nWhether this prop property can be NULL, optional values is true or false.\nsources.tags.props.nullValue\nNo\nIgnored when nullable is false. The value used to determine whether it is a NULL. The property is set to NULL when the value is equal to nullValue.\nsources.tags.props.alternativeIndices\nNo\nIgnored when nullable is false. The property is fetched from records according to the indices in order until not equal to nullValue.\nsources.tags.props.defaultValue\nNo\nIgnored when nullable is false. The property default value, when all the values obtained by index and alternativeIndices are nullValue.\nsources.edges.name\nYes\nThe edge type name.\nsources.edges.mode\nINSERT\nNo\nBatch operation types, including insert, update and delete. Optional values are INSERT, UPDATE and DELETE.\nsources.edges.filter.expr\nNo\nFilter the data and only import if the filter conditions are met. Supported comparison characters are ==, ! =, <, >, <= and >=. Logical operators supported are not (!) , and (&&) and or (||). For example (Record[0] == \"Mahinda\" or Record[0] == \"Michael\") and Record[3] == \"male\".\nsources.edges.src.id.type\nSTRING\nNo\nThe data type of the VID at the starting vertex on the edge.\nsources.edges.src.id.index\nYes\nThe column number in the data file corresponding to the VID at the starting vertex on the edge.\nsources.edges.dst.id.type\nSTRING\nNo\nThe data type of the VID at the destination vertex on the edge.\nsources.edges.dst.id.index\nYes\nThe column number in the data file corresponding to the VID at the destination vertex on the edge.\nsources.edges.rank.index\nNo\nThe column number in the data file corresponding to the rank on the edge.\nsources.edges.ignoreExistedIndex\ntrue\nNo\nWhether to enable IGNORE_EXISTED_INDEX, that is, do not update index after insertion vertex.\nsources.edges.props.name\nNo\nThe edge type property name, which must match the Tag property in the database.\nsources.edges.props.type\nSTRING\nNo\nProperty data type, supporting BOOL, INT, FLOAT, DOUBLE, STRING, TIME, TIMESTAMP, DATE, DATETIME, GEOGRAPHY, GEOGRAPHY(POINT), GEOGRAPHY(LINESTRING) and GEOGRAPHY(POLYGON).\nsources.edges.props.index\nNo\nThe property corresponds to the column number in the data file.\nsources.edges.props.nullable\nNo\nWhether this prop property can be NULL, optional values is true or false.\nsources.edges.props.nullValue\nNo\nIgnored when nullable is false. The value used to determine whether it is a NULL. The property is set to NULL when the value is equal to nullValue.\nsources.edges.props.defaultValue\nNo\nIgnored when nullable is false. The property default value, when all the values obtained by index and alternativeIndices are nullValue.","url":"https://docs.nebula-graph.io/3.6.0/import-export/use-importer/","type":"doc"},{"title":"What is NebulaGraph Exchange","content":"NebulaGraph Exchange (Exchange) is an Apache Spark™ application for bulk migration of cluster data to NebulaGraph in a distributed environment, supporting batch and streaming data migration in a variety of formats.\nExchange consists of Reader, Processor, and Writer. After Reader reads data from different sources and returns a DataFrame, the Processor iterates through each row of the DataFrame and obtains the corresponding value based on the mapping between fields in the configuration file. After iterating through the number of rows in the specified batch, Writer writes the captured data to the NebulaGraph at once. The following figure illustrates the process by which Exchange completes the data conversion and migration.\nEditions\nExchange has two editions, the Community Edition and the Enterprise Edition. The Community Edition is open source developed on GitHub. The Enterprise Edition supports not only the functions of the Community Edition but also adds additional features. For details, see Comparisons.\nScenarios\nExchange applies to the following scenarios:\nStreaming data from Kafka and Pulsar platforms, such as log files, online shopping data, activities of game players, information on social websites, financial transactions or geospatial services, and telemetry data from connected devices or instruments in the data center, are required to be converted into the vertex or edge data of the property graph and import them into the NebulaGraph database.\nBatch data, such as data from a time period, needs to be read from a relational database (such as MySQL) or a distributed file system (such as HDFS), converted into vertex or edge data for a property graph, and imported into the NebulaGraph database.\nA large volume of data needs to be generated into SST files that NebulaGraph can recognize and then imported into the NebulaGraph database.\nThe data saved in NebulaGraph needs to be exported.\nAdvantages\nExchange has the following advantages:\nHigh adaptability: It supports importing data into the NebulaGraph database in a variety of formats or from a variety of sources, making it easy to migrate data.\nSST import: It supports converting data from different sources into SST files for data import.\nSSL encryption: It supports establishing the SSL encryption between Exchange and NebulaGraph to ensure data security.\nResumable data import: It supports resumable data import to save time and improve data import efficiency.\nAsynchronous operation: An insert statement is generated in the source data and sent to the Graph service. Then the insert operation is performed.\nGreat flexibility: It supports importing multiple Tags and Edge types at the same time. Different Tags and Edge types can be from different data sources or in different formats.\nStatistics: It uses the accumulator in Apache Spark™ to count the number of successful and failed insert operations.\nEasy to use: It adopts the Human-Optimized Config Object Notation (HOCON) configuration file format and has an object-oriented style, which is easy to understand and operate.\nVersion compatibility\nExchange supports Spark versions 2.2.x, 2.4.x, and 3.x.x, which are named nebula-exchange_spark_2.2, nebula-exchange_spark_2.4, and nebula-exchange_spark_3.0 for different Spark versions.\nThe correspondence between the NebulaGraph Exchange version (the JAR version), the NebulaGraph core version and the Spark version is as follows.\nExchange version\nNebulaGraph version\nSpark version\nnebula-exchange_spark_3.0-3.0-SNAPSHOT.jar\nnightly\n3.3.x、3.2.x、3.1.x、3.0.x\nnebula-exchange_spark_2.4-3.0-SNAPSHOT.jar\nnightly\n2.4.x\nnebula-exchange_spark_2.2-3.0-SNAPSHOT.jar\nnightly\n2.2.x\nnebula-exchange_spark_3.0-3.4.0.jar\n3.x.x\n3.3.x、3.2.x、3.1.x、3.0.x\nnebula-exchange_spark_2.4-3.4.0.jar\n3.x.x\n2.4.x\nnebula-exchange_spark_2.2-3.4.0.jar\n3.x.x\n2.2.x\nnebula-exchange_spark_3.0-3.3.0.jar\n3.x.x\n3.3.x、3.2.x、3.1.x、3.0.x\nnebula-exchange_spark_2.4-3.3.0.jar\n3.x.x\n2.4.x\nnebula-exchange_spark_2.2-3.3.0.jar\n3.x.x\n2.2.x\nnebula-exchange_spark_3.0-3.0.0.jar\n3.x.x\n3.3.x、3.2.x、3.1.x、3.0.x\nnebula-exchange_spark_2.4-3.0.0.jar\n3.x.x\n2.4.x\nnebula-exchange_spark_2.2-3.0.0.jar\n3.x.x\n2.2.x\nnebula-exchange-2.6.3.jar\n2.6.1、2.6.0\n2.4.x\nnebula-exchange-2.6.2.jar\n2.6.1、2.6.0\n2.4.x\nnebula-exchange-2.6.1.jar\n2.6.1、2.6.0\n2.4.x\nnebula-exchange-2.6.0.jar\n2.6.1、2.6.0\n2.4.x\nnebula-exchange-2.5.2.jar\n2.5.1、2.5.0\n2.4.x\nnebula-exchange-2.5.1.jar\n2.5.1、2.5.0\n2.4.x\nnebula-exchange-2.5.0.jar\n2.5.1、2.5.0\n2.4.x\nnebula-exchange-2.1.0.jar\n2.0.1、2.0.0\n2.4.x\nnebula-exchange-2.0.1.jar\n2.0.1、2.0.0\n2.4.x\nnebula-exchange-2.0.0.jar\n2.0.1、2.0.0\n2.4.x\nData source\nExchange 3.6.0 supports converting data from the following formats or sources into vertexes and edges that NebulaGraph can recognize, and then importing them into NebulaGraph in the form of nGQL statements:\nData stored in HDFS or locally:\nApache Parquet\nApache ORC\nJSON\nCSV\nApache HBase™\nData repository:\nHive\nMaxCompute\nGraph database: Neo4j (Client version 2.4.5-M1)\nRelational database:\nMySQL\nPostgreSQL\nOracle\nColumn database: ClickHouse\nStream processing software platform: Apache Kafka®\nPublish/Subscribe messaging platform: Apache Pulsar 2.4.5\nJDBC\nIn addition to importing data as nGQL statements, Exchange supports generating SST files for data sources and then importing SST files via Console.\nRelease note\nRelease","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/about-exchange/ex-ug-what-is-exchange/","type":"doc"},{"title":"Limitations","content":"This topic describes some of the limitations of using Exchange 3.x.\nEnvironment\nExchange 3.x supports the following operating systems:\nCentOS 7\nmacOS\nSoftware dependencies\nTo ensure the healthy operation of Exchange, ensure that the following software has been installed on the machine:\nJava version 1.8\nScala version 2.10.7, 2.11.12, or 2.12.10\nApache Spark. The requirements for Spark versions when using Exchange to export data from data sources are as follows. In the following table, Y means that the corresponding Spark version is supported, and N means not supported.\nData source\nSpark 2.2\nSpark 2.4\nSpark 3\nCSV file\nJSON file\nORC file\nParquet file\nHBase\nMySQL\nPostgreSQL\nOracle\nClickHouse\nNeo4j\nHive\nMaxCompute\nPulsar\nUntested\nKafka\nUntested\nNebulaGraph\nHadoop Distributed File System (HDFS) needs to be deployed in the following scenarios:\nMigrate HDFS data\nGenerate SST files","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/about-exchange/ex-ug-limitations/","type":"doc"},{"title":"Get Exchange","content":"This topic introduces how to get the JAR file of NebulaGraph Exchange.\nDownload the JAR file directly\nThe JAR file of Exchange Community Edition can be downloaded directly.\nTo download Exchange Enterprise Edition, contact us.\nGet the JAR file by compiling the source code\nYou can get the JAR file of Exchange Community Edition by compiling the source code. The following introduces how to compile the source code of Exchange.\nPrerequisites\nInstall Maven.\nInstall the correct version of Apache Spark. Exporting data from different sources requires different Spark versions. For more information, see Software dependencies.\nSteps\nClone the repository nebula-exchange in the / directory.\ngit clone -b release-3.6 https://github.com/vesoft-inc/nebula-exchange.git\nSwitch to the directory nebula-exchange.\ncd nebula-exchange\nPackage NebulaGraph Exchange. Run the following command based on the Spark version:\nFor Spark 2.2：\nmvn clean package -Dmaven.test.skip=true -Dgpg.skip -Dmaven.javadoc.skip=true \\\n-pl nebula-exchange_spark_2.2 -am -Pscala-2.11 -Pspark-2.2\nFor Spark 2.4：\nmvn clean package -Dmaven.test.skip=true -Dgpg.skip -Dmaven.javadoc.skip=true \\\n-pl nebula-exchange_spark_2.4 -am -Pscala-2.11 -Pspark-2.4\nFor Spark 3.0：\nmvn clean package -Dmaven.test.skip=true -Dgpg.skip -Dmaven.javadoc.skip=true \\\n-pl nebula-exchange_spark_3.0 -am -Pscala-2.12 -Pspark-3.0\nAfter the compilation is successful, you can find the nebula-exchange_spark_x.x-release-3.6.jar file in the nebula-exchange_spark_x.x/target/ directory. x.x indicates the Spark version, for example, 2.4.\nWhen migrating data, you can refer to configuration file target/classes/application.conf.\nFailed to download the dependency package\nIf downloading dependencies fails when compiling:\nCheck the network settings and ensure that the network is normal.\nModify the mirror part of Maven installation directory libexec/conf/settings.xml:\n\n alimaven\n central\n aliyun maven\n http://maven.aliyun.com/nexus/content/repositories/central/\n","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/ex-ug-compile/","type":"doc"},{"title":"Options for import","content":"After editing the configuration file, run the following commands to import specified source data into the NebulaGraph database.\nImport data\n/bin/spark-submit --master \"spark://HOST:PORT\" --class com.vesoft.nebula.exchange.Exchange -c \nThe following table lists command parameters.\nParameter\nRequired\nDefault value\nDescription\n--class \nYes\nSpecify the main class of the driver.\n--master \nYes\nSpecify the URL of the master process in a Spark cluster. For more information, see master-urls. Optional values are:local: Local Mode. Run Spark applications on a single thread. Suitable for importing small data sets in a test environment.yarn: Run Spark applications on a YARN cluster. Suitable for importing large data sets in a production environment.spark://HOST:PORT: Connect to the specified Spark standalone cluster.mesos://HOST:PORT: Connect to the specified Mesos cluster.k8s://HOST:PORT: Connect to the specified Kubernetes cluster.\n-c/--config \nYes\nSpecify the path of the configuration file.\n-h/--hive \nNo\nfalse\nSpecify whether importing Hive data is supported.\n-D/--dry \nNo\nfalse\nSpecify whether to check the format of the configuration file. This parameter is used to check the format of the configuration file only, it does not check the validity of tags and edges configurations and does not import data. Don't add this parameter if you need to import data.\n-r/--reload\nNo\nSpecify the path of the reload file that needs to be reloaded.\nFor more Spark parameter configurations, see Spark Configuration.\nImport the reload file\nIf some data fails to be imported during the import, the failed data will be stored in the reload file. Use the parameter -r to import the data in reload file.\n/bin/spark-submit --master \"spark://HOST:PORT\" --class com.vesoft.nebula.exchange.Exchange -c -r \"\" \nIf the import still fails, go to Official Forum for consultation.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/parameter-reference/ex-ug-para-import-command/","type":"doc"},{"title":"Parameters in the configuration file","content":"This topic describes how to configure the file application.conf when users use NebulaGraph Exchange.\nBefore configuring the application.conf file, it is recommended to copy the file name application.conf and then edit the file name according to the file type of a data source. For example, change the file name to csv_application.conf if the file type of the data source is CSV.\nThe application.conf file contains the following content types:\nSpark configurations\nHive configurations (optional)\nNebulaGraph configurations\nVertex configurations\nEdge configurations\nSpark configurations\nThis topic lists only some Spark parameters. For more information, see Spark Configuration.\nParameter\nType\nDefault value\nRequired\nDescription\nspark.app.name\nstring\nNo\nThe drive name in Spark.\nspark.driver.cores\nint\nNo\nThe number of CPU cores used by a driver, only applicable to a cluster mode.\nspark.driver.maxResultSize\nstring\n1G\nNo\nThe total size limit (in bytes) of the serialized results of all partitions in a single Spark operation (such as collect). The minimum value is 1M, and 0 means unlimited.\nspark.executor.memory\nstring\n1G\nNo\nThe amount of memory used by a Spark driver which can be specified in units, such as 512M or 1G.\nspark.cores.max\nint\n16\nNo\nThe maximum number of CPU cores of applications requested across clusters (rather than from each node) when a driver runs in a coarse-grained sharing mode on a standalone cluster or a Mesos cluster. The default value is spark.deploy.defaultCores on a Spark standalone cluster manager or the value of the infinite parameter (all available cores) on Mesos.\nHive configurations (optional)\nUsers only need to configure parameters for connecting to Hive if Spark and Hive are deployed in different clusters. Otherwise, please ignore the following configurations.\nParameter\nType\nDefault value\nRequired\nDescription\nhive.warehouse\nstring\nYes\nThe warehouse path in HDFS. Enclose the path in double quotes and start with hdfs://.\nhive.connectionURL\nstring\nYes\nThe URL of a JDBC connection. For example, \"jdbc:mysql://127.0.0.1:3306/hive_spark?characterEncoding=UTF-8\".\nhive.connectionDriverName\nstring\n\"com.mysql.jdbc.Driver\"\nYes\nThe driver name.\nhive.connectionUserName\nlist[string]\nYes\nThe username for connections.\nhive.connectionPassword\nlist[string]\nYes\nThe account password.\nNebulaGraph configurations\nParameter\nType\nDefault value\nRequired\nDescription\nnebula.address.graph\nlist[string]\n[\"127.0.0.1:9669\"]\nYes\nThe addresses of all Graph services, including IPs and ports, separated by commas (,). Example: [\"ip1:port1\",\"ip2:port2\",\"ip3:port3\"].\nnebula.address.meta\nlist[string]\n[\"127.0.0.1:9559\"]\nYes\nThe addresses of all Meta services, including IPs and ports, separated by commas (,). Example: [\"ip1:port1\",\"ip2:port2\",\"ip3:port3\"].\nnebula.user\nstring\nYes\nThe username with write permissions for NebulaGraph.\nnebula.pswd\nstring\nYes\nThe account password.\nnebula.space\nstring\nYes\nThe name of the graph space where data needs to be imported.\nnebula.ssl.enable.graph\nbool\nfalse\nYes\nEnables the SSL encryption between Exchange and Graph services. If the value is true, the SSL encryption is enabled and the following SSL parameters take effect. If Exchange is run on a multi-machine cluster, you need to store the corresponding files in the same path on each machine when setting the following SSL-related paths.\nnebula.ssl.sign\nstring\nca\nYes\nSpecifies the SSL sign. Optional values are ca and self.\nnebula.ssl.ca.param.caCrtFilePath\nstring\nSpecifies the storage path of the CA certificate. It takes effect when the value of nebula.ssl.sign is ca.\nnebula.ssl.ca.param.crtFilePath\nstring\n\"/path/crtFilePath\"\nYes\nSpecifies the storage path of the CRT certificate. It takes effect when the value of nebula.ssl.sign is ca.\nnebula.ssl.ca.param.keyFilePath\nstring\n\"/path/keyFilePath\"\nYes\nSpecifies the storage path of the key file. It takes effect when the value of nebula.ssl.sign is ca.\nnebula.ssl.self.param.crtFilePath\nstring\n\"/path/crtFilePath\"\nYes\nSpecifies the storage path of the CRT certificate. It takes effect when the value of nebula.ssl.sign is self.\nnebula.ssl.self.param.keyFilePath\nstring\n\"/path/keyFilePath\"\nYes\nSpecifies the storage path of the key file. It takes effect when the value of nebula.ssl.sign is self.\nnebula.ssl.self.param.password\nstring\n\"nebula\"\nYes\nSpecifies the storage path of the password. It takes effect when the value of nebula.ssl.sign is self.\nnebula.path.local\nstring\n\"/tmp\"\nNo\nThe local SST file path which needs to be set when users import SST files.\nnebula.path.remote\nstring\n\"/sst\"\nNo\nThe remote SST file path which needs to be set when users import SST files.\nnebula.path.hdfs.namenode\nstring\n\"hdfs://name_node:9000\"\nNo\nThe NameNode path which needs to be set when users import SST files.\nnebula.connection.timeout\nint\n3000\nNo\nThe timeout set for Thrift connections. Unit: ms.\nnebula.connection.retry\nint\nNo\nRetries set for Thrift connections.\nnebula.execution.retry\nint\nNo\nRetries set for executing nGQL statements.\nnebula.error.max\nint\n32\nNo\nThe maximum number of failures during the import process. When the number of failures reaches the maximum, the Spark job submitted will stop automatically .\nnebula.error.output\nstring\n/tmp/errors\nNo\nThe path to output error logs. Failed nGQL statement executions are saved in the error log.\nnebula.rate.limit\nint\n1024\nNo\nThe limit on the number of tokens in the token bucket when importing data.\nnebula.rate.timeout\nint\n1000\nNo\nThe timeout period for getting tokens from a token bucket. Unit: milliseconds.\nVertex configurations\nFor different data sources, the vertex configurations are different. There are many general parameters and some specific parameters. General parameters and specific parameters of different data sources need to be configured when users configure vertices.\nGeneral parameters\nParameter\nType\nDefault value\nRequired\nDescription\ntags.name\nstring\nYes\nThe tag name defined in NebulaGraph.\ntags.type.source\nstring\nYes\nSpecify a data source. For example, csv.\ntags.type.sink\nstring\nclient\nYes\nSpecify an import method. Optional values are client and SST.\ntags.writeMode\nstring\nINSERT\nNo\nTypes of batch operations on data, including batch inserts, updates, and deletes. Optional values are INSERT, UPDATE, DELETE.\ntags.deleteEdge\nstring\nfalse\nNo\nWhether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when tags.writeMode is DELETE.\ntags.fields\nlist[string]\nYes\nThe header or column name of the column corresponding to properties. If there is a header or a column name, please use that name directly. If a CSV file does not have a header, use the form of [_c0, _c1, _c2] to represent the first column, the second column, the third column, and so on.\ntags.nebula.fields\nlist[string]\nYes\nProperty names defined in NebulaGraph, the order of which must correspond to tags.fields. For example, [_c1, _c2] corresponds to [name, age], which means that values in the second column are the values of the property name, and values in the third column are the values of the property age.\ntags.vertex.field\nstring\nYes\nThe column of vertex IDs. For example, when a CSV file has no header, users can use _c0 to indicate values in the first column are vertex IDs.\ntags.vertex.udf.separator\nstring\nNo\nSupport merging multiple columns by custom rules. This parameter specifies the join character.\ntags.vertex.udf.oldColNames\nlist\nNo\nSupport merging multiple columns by custom rules. This parameter specifies the names of the columns to be merged. Multiple columns are separated by commas.\ntags.vertex.udf.newColName\nstring\nNo\nSupport merging multiple columns by custom rules. This parameter specifies the new column name.\ntags.vertex.prefix\nstring\nNo\nAdd the specified prefix to the VID. For example, if the VID is 12345, adding the prefix tag1 will result in tag1_12345. The underscore cannot be modified.\ntags.vertex.policy\nstring\nNo\nSupports only the value hash. Performs hashing operations on VIDs of type string.\ntags.batch\nint\n256\nYes\nThe maximum number of vertices written into NebulaGraph in a single batch.\ntags.partition\nint\n32\nYes\nThe number of partitions to be created when the data is written to NebulaGraph. If tags.partition ≤ 1, the number of partitions to be created in NebulaGraph is the same as that in the data source.\nSpecific parameters of Parquet/JSON/ORC data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.path\nstring\nYes\nThe path of vertex data files in HDFS. Enclose the path in double quotes and start with hdfs://.\nSpecific parameters of CSV data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.path\nstring\nYes\nThe path of vertex data files in HDFS. Enclose the path in double quotes and start with hdfs://.\ntags.separator\nstring\nYes\nThe separator. The default value is a comma (,). For special characters, such as the control character ^A, you can use ASCII octal \\001 or UNICODE encoded hexadecimal \\u0001, for the control character ^B, use ASCII octal \\002 or UNICODE encoded hexadecimal \\u0002, for the control character ^C, use ASCII octal \\003 or UNICODE encoded hexadecimal \\u0003.\ntags.header\nbool\ntrue\nYes\nWhether the file has a header.\nSpecific parameters of Hive data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.exec\nstring\nYes\nThe statement to query data sources. For example, select name,age from mooc.users.\nSpecific parameters of MaxCompute data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.table\nstring\nYes\nThe table name of the MaxCompute.\ntags.project\nstring\nYes\nThe project name of the MaxCompute.\ntags.odpsUrl\nstring\nYes\nThe odpsUrl of the MaxCompute service. For more information about odpsUrl, see Endpoints.\ntags.tunnelUrl\nstring\nYes\nThe tunnelUrl of the MaxCompute service. For more information about tunnelUrl, see Endpoints.\ntags.accessKeyId\nstring\nYes\nThe accessKeyId of the MaxCompute service.\ntags.accessKeySecret\nstring\nYes\nThe accessKeySecret of the MaxCompute service.\ntags.partitionSpec\nstring\nNo\nPartition descriptions of MaxCompute tables.\ntags.sentence\nstring\nNo\nStatements to query data sources. The table name in the SQL statement is the same as the value of the table above.\nSpecific parameters of Neo4j data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.exec\nstring\nYes\nStatements to query data sources. For example: match (n:label) return n.neo4j-field-0.\ntags.server\nstring\n\"bolt://127.0.0.1:7687\"\nYes\nThe server address of Neo4j.\ntags.user\nstring\nYes\nThe Neo4j username with read permissions.\ntags.password\nstring\nYes\nThe account password.\ntags.database\nstring\nYes\nThe name of the database where source data is saved in Neo4j.\ntags.check_point_path\nstring\n/tmp/test\nNo\nThe directory set to import progress information, which is used for resuming transfers. If not set, the resuming transfer is disabled.\nSpecific parameters of MySQL/PostgreSQL data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.host\nstring\nYes\nThe MySQL/PostgreSQL server address.\ntags.port\nstring\nYes\nThe MySQL/PostgreSQL server port.\ntags.database\nstring\nYes\nThe database name.\ntags.table\nstring\nYes\nThe name of a table used as a data source.\ntags.user\nstring\nYes\nThe MySQL/PostgreSQL username with read permissions.\ntags.password\nstring\nYes\nThe account password.\ntags.sentence\nstring\nYes\nStatements to query data sources. For example: \"select teamid, name from team order by teamid\".\nSpecific parameters of Oracle data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.url\nstring\nYes\nThe Oracle server address.\ntags.driver\nstring\nYes\nThe Oracle driver address.\ntags.user\nstring\nYes\nThe Oracle username with read permissions.\ntags.password\nstring\nYes\nThe account password.\ntags.table\nstring\nYes\nThe name of a table used as a data source.\ntags.sentence\nstring\nYes\nStatements to query data sources. For example: \"select playerid, name, age from player\".\nSpecific parameters of ClickHouse data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.url\nstring\nYes\nThe JDBC URL of ClickHouse.\ntags.user\nstring\nYes\nThe ClickHouse username with read permissions.\ntags.password\nstring\nYes\nThe account password.\ntags.numPartition\nstring\nYes\nThe number of ClickHouse partitions.\ntags.sentence\nstring\nYes\nStatements to query data sources.\nSpecific parameters of Hbase data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.host\nstring\n127.0.0.1\nYes\nThe Hbase server address.\ntags.port\nstring\n2181\nYes\nThe Hbase server port.\ntags.table\nstring\nYes\nThe name of a table used as a data source.\ntags.columnFamily\nstring\nYes\nThe column family to which a table belongs.\nSpecific parameters of Pulsar data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.service\nstring\n\"pulsar://localhost:6650\"\nYes\nThe Pulsar server address.\ntags.admin\nstring\n\"http://localhost:8081\"\nYes\nThe admin URL used to connect pulsar.\ntags.options.\nstring\nYes\nOptions offered by Pulsar, which can be configured by choosing one from topic, topics, and topicsPattern.\ntags.interval.seconds\nint\n10\nYes\nThe interval for reading messages. Unit: seconds.\nSpecific parameters of Kafka data sources\nParameter\nType\nDefault value\nRequired\nDescription\ntags.service\nstring\nYes\nThe Kafka server address.\ntags.topic\nstring\nYes\nThe message type.\ntags.interval.seconds\nint\n10\nYes\nThe interval for reading messages. Unit: seconds.\nSpecific parameters for generating SST files\nParameter\nType\nDefault value\nRequired\nDescription\ntags.path\nstring\nYes\nThe path of the source file specified to generate SST files.\ntags.repartitionWithNebula\nbool\ntrue\nNo\nWhether to repartition data based on the number of partitions of graph spaces in NebulaGraph when generating the SST file. Enabling this function can reduce the time required to DOWNLOAD and INGEST SST files.\nEdge configurations\nFor different data sources, configurations of edges are also different. There are general parameters and some specific parameters. General parameters and specific parameters of different data sources need to be configured when users configure edges.\nFor the specific parameters of different data sources for edge configurations, please refer to the introduction of specific parameters of different data sources above, and pay attention to distinguishing tags and edges.\nGeneral parameters\nParameter\nType\nDefault value\nRequired\nDescription\nedges.name\nstring\nYes\nThe edge type name defined in NebulaGraph.\nedges.type.source\nstring\nYes\nThe data source of edges. For example, csv.\nedges.type.sink\nstring\nclient\nYes\nThe method specified to import data. Optional values are client and SST.\nedges.writeMode\nstring\nINSERT\nNo\nTypes of batch operations on data, including batch inserts, updates, and deletes. Optional values are INSERT, UPDATE, DELETE.\nedges.fields\nlist[string]\nYes\nThe header or column name of the column corresponding to properties. If there is a header or column name, please use that name directly. If a CSV file does not have a header, use the form of [_c0, _c1, _c2] to represent the first column, the second column, the third column, and so on.\nedges.nebula.fields\nlist[string]\nYes\nEdge names defined in NebulaGraph, the order of which must correspond to edges.fields. For example, [_c2, _c3] corresponds to [start_year, end_year], which means that values in the third column are the values of the start year, and values in the fourth column are the values of the end year.\nedges.source.field\nstring\nYes\nThe column of source vertices of edges. For example, _c0 indicates a value in the first column that is used as the source vertex of an edge.\nedges.source.prefix\nstring\nNo\nAdd the specified prefix to the VID. For example, if the VID is 12345, adding the prefix tag1 will result in tag1_12345. The underscore cannot be modified.\nedges.source.policy\nstring\nNo\nSupports only the value hash. Performs hashing operations on VIDs of type string.\nedges.target.field\nstring\nYes\nThe column of destination vertices of edges. For example, _c0 indicates a value in the first column that is used as the destination vertex of an edge.\nedges.target.prefix\nstring\nNo\nAdd the specified prefix to the VID. For example, if the VID is 12345, adding the prefix tag1 will result in tag1_12345. The underscore cannot be modified.\nedges.target.policy\nstring\nNo\nSupports only the value hash. Performs hashing operations on VIDs of type string.\nedges.ranking\nint\nNo\nThe column of rank values. If not specified, all rank values are 0 by default.\nedges.batch\nint\n256\nYes\nThe maximum number of edges written into NebulaGraph in a single batch.\nedges.partition\nint\n32\nYes\nThe number of partitions to be created when the data is written to NebulaGraph. If edges.partition ≤ 1, the number of partitions to be created in NebulaGraph is the same as that in the data source.\nSpecific parameters for generating SST files\nParameter\nType\nDefault value\nRequired\nDescription\nedges.path\nstring\nYes\nThe path of the source file specified to generate SST files.\nedges.repartitionWithNebula\nbool\ntrue\nNo\nWhether to repartition data based on the number of partitions of graph spaces in NebulaGraph when generating the SST file. Enabling this function can reduce the time required to DOWNLOAD and INGEST SST files.\nSpecific parameters of NebulaGraph\nParameter\nType\nDefault value\nRequired\nDescription\nedges.path\nstring\n\"hdfs://namenode:9000/path/edge\"\nYes\nSpecifies the storage path of the CSV file. You need to set a new path and Exchange will automatically create the path you set. If you store the data to the HDFS server, the path format is the same as the default value, such as \"hdfs://192.168.8.177:9000/edge/follow\". If you store the data to the local, the path format is \"file:///path/edge\", such as \"file:///home/nebula/edge/follow\". If there are multiple Edges, different directories must be set for each Edge.\nedges.noField\nbool\nfalse\nYes\nIf the value is true, source vertex IDs, destination vertex IDs, and ranks will be exported, not the property data. If the vaue is false, ranks, source vertex IDs, destination vertex IDs, ranks, and the property data will be exported.\nedges.return.fields\nlist\n[]\nYes\nSpecifies the properties to be exported. For example, to export start_year and end_year, you need to set the parameter value to [\"start_year\",\"end_year\"]. This parameter only takes effect when the value of edges.noField is false.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/parameter-reference/ex-ug-parameter/","type":"doc"},{"title":"Import data from CSV files","content":"This topic provides an example of how to use Exchange to import NebulaGraph data stored in HDFS or local CSV files.\nData set\nThis topic takes the basketballplayer dataset as an example.\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nHadoop: 2.9.2, pseudo-distributed deployment\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nIf files are stored in HDFS, ensure that the Hadoop service is running normally.\nIf files are stored locally and NebulaGraph is a cluster architecture, you need to place the files in the same directory locally on each machine in the cluster.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Process CSV files\nConfirm the following information:\nProcess CSV files to meet Schema requirements.\nObtain the CSV file storage path.\nStep 3: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set CSV data source configuration. In this example, the copied file is called csv_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n executor: {\n memory:1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n # Specify the Tag name defined in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to CSV.\n source: csv\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Specify the path to the CSV file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example: \"hdfs://ip:port/xx/xx\".\n # If the file is stored locally, use double quotation marks to enclose the file path, starting with file://. For example: \"file:///tmp/xx.csv\".\n path: \"hdfs://192.168.*.*:9000/data/vertex_player.csv\"\n # If the CSV file does not have a header, use [_c0, _c1, _c2, ..., _cn] to represent its header and indicate the columns as the source of the property values.\n # If the CSV file has headers, use the actual column names.\n fields: [_c1, _c2]\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [age, name]\n # Specify a column of data in the table as the source of vertex VID in the NebulaGraph.\n # The value of vertex must be the same as the column names in the above fields or csv.fields.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n vertex: {\n field:_c0\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # The delimiter specified. The default value is comma.\n separator: \",\"\n # If the CSV file has a header, set the header to true.\n # If the CSV file does not have a header, set the header to false. The default value is false.\n header: false\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag Team.\n name: team\n type: {\n source: csv\n sink: client\n path: \"hdfs://192.168.*.*:9000/data/vertex_team.csv\"\n fields: [_c1]\n nebula.fields: [name]\n vertex: {\n field:_c0\n separator: \",\"\n header: false\n batch: 256\n partition: 32\n # If more vertexes need to be added, refer to the previous configuration to add them.\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # Specify the Edge Type name defined in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to CSV.\n source: csv\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Specify the path to the CSV file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example: \"hdfs://ip:port/xx/xx\".\n # If the file is stored locally, use double quotation marks to enclose the file path, starting with file://. For example: \"file:///tmp/xx.csv\".\n path: \"hdfs://192.168.*.*:9000/data/edge_follow.csv\"\n # If the CSV file does not have a header, use [_c0, _c1, _c2, ..., _cn] to represent its header and indicate the columns as the source of the property values.\n # If the CSV file has headers, use the actual column names.\n fields: [_c2]\n # Specify the column names in the edge table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [degree]\n # Specify a column as the source for the source and destination vertexes.\n # The value of vertex must be the same as the column names in the above fields or csv.fields.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n source: {\n field: _c0\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: _c1\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # The delimiter specified. The default value is comma.\n separator: \",\"\n # Specify a column as the source of the rank (optional).\n #ranking: rank\n # If the CSV file has a header, set the header to true.\n # If the CSV file does not have a header, set the header to false. The default value is false.\n header: false\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge Type serve.\n name: serve\n type: {\n source: csv\n sink: client\n path: \"hdfs://192.168.*.*:9000/data/edge_serve.csv\"\n fields: [_c2,_c3]\n nebula.fields: [start_year, end_year]\n source: {\n field: _c0\n target: {\n field: _c1\n separator: \",\"\n header: false\n batch: 256\n partition: 32\n # If more edges need to be added, refer to the previous configuration to add them.\nStep 4: Import data into NebulaGraph\nRun the following command to import CSV data into NebulaGraph. For descriptions of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/csv_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 5: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 6: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-csv/","type":"doc"},{"title":"Import data from JSON files","content":"This topic provides an example of how to use Exchange to import NebulaGraph data stored in HDFS or local JSON files.\nData set\nThis topic takes the basketballplayer dataset as an example. Some sample data are as follows:\nplayer\n{\"id\":\"player100\",\"age\":42,\"name\":\"Tim Duncan\"}\n{\"id\":\"player101\",\"age\":36,\"name\":\"Tony Parker\"}\n{\"id\":\"player102\",\"age\":33,\"name\":\"LaMarcus Aldridge\"}\n{\"id\":\"player103\",\"age\":32,\"name\":\"Rudy Gay\"}\n...\nteam\n{\"id\":\"team200\",\"name\":\"Warriors\"}\n{\"id\":\"team201\",\"name\":\"Nuggets\"}\n...\nfollow\n{\"src\":\"player100\",\"dst\":\"player101\",\"degree\":95}\n{\"src\":\"player101\",\"dst\":\"player102\",\"degree\":90}\n...\nserve\n{\"src\":\"player100\",\"dst\":\"team204\",\"start_year\":\"1997\",\"end_year\":\"2016\"}\n{\"src\":\"player101\",\"dst\":\"team204\",\"start_year\":\"1999\",\"end_year\":\"2018\"}\n...\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nHadoop: 2.9.2, pseudo-distributed deployment\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nIf files are stored in HDFS, ensure that the Hadoop service is running properly.\nIf files are stored locally and NebulaGraph is a cluster architecture, you need to place the files in the same directory locally on each machine in the cluster.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Process JSON files\nConfirm the following information:\nProcess JSON files to meet Schema requirements.\nObtain the JSON file storage path.\nStep 3: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set JSON data source configuration. In this example, the copied file is called json_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n executor: {\n memory:1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and all Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n # Specify the Tag name defined in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to JSON.\n source: json\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Specify the path to the JSON file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example, \"hdfs://ip:port/xx/xx\".\n # If the file is stored locally, use double quotation marks to enclose the file path, starting with file://. For example, \"file:///tmp/xx.json\".\n path: \"hdfs://192.168.*.*:9000/data/vertex_player.json\"\n # Specify the key name in the JSON file in fields, and its corresponding value will serve as the data source for the properties specified in the NebulaGraph.\n # If multiple column names need to be specified, separate them by commas.\n fields: [age,name]\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [age, name]\n # Specify a column of data in the table as the source of vertex VID in the NebulaGraph.\n # The value of vertex must be the same as that in the JSON file.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n vertex: {\n field:id\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag Team.\n name: team\n type: {\n source: json\n sink: client\n path: \"hdfs://192.168.*.*:9000/data/vertex_team.json\"\n fields: [name]\n nebula.fields: [name]\n vertex: {\n field:id\n batch: 256\n partition: 32\n # If more vertexes need to be added, refer to the previous configuration to add them.\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # Specify the Edge Type name defined in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to JSON.\n source: json\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Specify the path to the JSON file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example, \"hdfs://ip:port/xx/xx\".\n # If the file is stored locally, use double quotation marks to enclose the file path, starting with file://. For example, \"file:///tmp/xx.json\".\n path: \"hdfs://192.168.*.*:9000/data/edge_follow.json\"\n # Specify the key name in the JSON file in fields, and its corresponding value will serve as the data source for the properties specified in the NebulaGraph.\n # If multiple column names need to be specified, separate them by commas.\n fields: [degree]\n # Specify the column names in the edge table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [degree]\n # Specify a column as the source for the source and destination vertexes.\n # The value of vertex must be the same as that in the JSON file.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n source: {\n field: src\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: dst\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge Type serve.\n name: serve\n type: {\n source: json\n sink: client\n path: \"hdfs://192.168.*.*:9000/data/edge_serve.json\"\n fields: [start_year,end_year]\n nebula.fields: [start_year, end_year]\n source: {\n field: src\n target: {\n field: dst\n batch: 256\n partition: 32\n # If more edges need to be added, refer to the previous configuration to add them.\nStep 4: Import data into NebulaGraph\nRun the following command to import JSON data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-echange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/json_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 5: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 6: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-json/","type":"doc"},{"title":"Import data from ORC files","content":"This topic provides an example of how to use Exchange to import NebulaGraph data stored in HDFS or local ORC files.\nData set\nThis topic takes the basketballplayer dataset as an example.\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nHadoop: 2.9.2, pseudo-distributed deployment\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nIf files are stored in HDFS, ensure that the Hadoop service is running properly.\nIf files are stored locally and NebulaGraph is a cluster architecture, you need to place the files in the same directory locally on each machine in the cluster.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Process ORC files\nConfirm the following information:\nProcess ORC files to meet Schema requirements.\nObtain the ORC file storage path.\nStep 3: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set ORC data source configuration. In this example, the copied file is called orc_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n executor: {\n memory:1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and all Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n name: player\n type: {\n # Specify the data source file format to ORC.\n source: orc\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Specify the path to the ORC file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example, \"hdfs://ip:port/xx/xx\".\n # If the file is stored locally, use double quotation marks to enclose the file path, starting with file://. For example, \"file:///tmp/xx.orc\".\n path: \"hdfs://192.168.*.*:9000/data/vertex_player.orc\"\n # Specify the key name in the ORC file in fields, and its corresponding value will serve as the data source for the properties specified in the NebulaGraph.\n # If multiple values need to be specified, separate them with commas.\n fields: [age,name]\n # Specify the property names defined in NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [age, name]\n # Specify a column of data in the table as the source of VIDs in the NebulaGraph.\n # The value of vertex must be consistent with the field in the ORC file.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n vertex: {\n field:id\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag team.\n name: team\n type: {\n source: orc\n sink: client\n path: \"hdfs://192.168.*.*:9000/data/vertex_team.orc\"\n fields: [name]\n nebula.fields: [name]\n vertex: {\n field:id\n batch: 256\n partition: 32\n # If more vertexes need to be added, refer to the previous configuration to add them.\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # Specify the Edge Type name defined in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to ORC.\n source: orc\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Specify the path to the ORC file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example, \"hdfs://ip:port/xx/xx\".\n # If the file is stored locally, use double quotation marks to enclose the file path, starting with file://. For example, \"file:///tmp/xx.orc\".\n path: \"hdfs://192.168.*.*:9000/data/edge_follow.orc\"\n # Specify the key name in the ORC file in fields, and its corresponding value will serve as the data source for the properties specified in the NebulaGraph.\n # If multiple values need to be specified, separate them with commas.\n fields: [degree]\n # Specify the property names defined in NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [degree]\n # Specify a column as the source for the source and destination vertexes.\n # The value of vertex must be consistent with the field in the ORC file.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n source: {\n field: src\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: dst\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge type serve.\n name: serve\n type: {\n source: orc\n sink: client\n path: \"hdfs://192.168.*.*:9000/data/edge_serve.orc\"\n fields: [start_year,end_year]\n nebula.fields: [start_year, end_year]\n source: {\n field: src\n target: {\n field: dst\n batch: 256\n partition: 32\n # If more edges need to be added, refer to the previous configuration to add them.\nStep 4: Import data into NebulaGraph\nRun the following command to import ORC data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/orc_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 5: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 6: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-orc/","type":"doc"},{"title":"Import data from Parquet files","content":"This topic provides an example of how to use Exchange to import NebulaGraph data stored in HDFS or local Parquet files.\nData set\nThis topic takes the basketballplayer dataset as an example.\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nHadoop: 2.9.2, pseudo-distributed deployment\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nIf files are stored in HDFS, ensure that the Hadoop service is running properly.\nIf files are stored locally and NebulaGraph is a cluster architecture, you need to place the files in the same directory locally on each machine in the cluster.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Process Parquet files\nConfirm the following information:\nProcess Parquet files to meet Schema requirements.\nObtain the Parquet file storage path.\nStep 3: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set Parquet data source configuration. In this example, the copied file is called parquet_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n executor: {\n memory:1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and all Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n # Specify the Tag name defined in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to Parquet.\n source: parquet\n # Specifies how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Specify the path to the Parquet file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example, \"hdfs://ip:port/xx/xx\".\n # If the file is stored locally, use double quotation marks to enclose the file path, starting with file://. For example, \"file:///tmp/xx.parquet\".\n path: \"hdfs://192.168.*.13:9000/data/vertex_player.parquet\"\n # Specify the key name in the Parquet file in fields, and its corresponding value will serve as the data source for the properties specified in the NebulaGraph.\n # If multiple values need to be specified, separate them with commas.\n fields: [age,name]\n # Specify the property name defined in NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [age, name]\n # Specify a column of data in the table as the source of VIDs in the NebulaGraph.\n # The value of vertex must be consistent with the field in the Parquet file.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n vertex: {\n field:id\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag team.\n name: team\n type: {\n source: parquet\n sink: client\n path: \"hdfs://192.168.11.13:9000/data/vertex_team.parquet\"\n fields: [name]\n nebula.fields: [name]\n vertex: {\n field:id\n batch: 256\n partition: 32\n # If more vertexes need to be added, refer to the previous configuration to add them.\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # Specify the Edge Type name defined in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to Parquet.\n source: parquet\n # Specifies how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Specify the path to the Parquet file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example, \"hdfs://ip:port/xx/xx\".\n # If the file is stored locally, use double quotation marks to enclose the file path, starting with file://. For example, \"file:///tmp/xx.parquet\".\n path: \"hdfs://192.168.11.13:9000/data/edge_follow.parquet\"\n # Specify the key name in the Parquet file in fields, and its corresponding value will serve as the data source for the properties specified in the NebulaGraph.\n # If multiple values need to be specified, separate them with commas.\n fields: [degree]\n # Specify the property name defined in NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [degree]\n # Specify a column as the source for the source and destination vertexes.\n # The values of vertex must be consistent with the fields in the Parquet file.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n source: {\n field: src\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: dst\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge type serve.\n name: serve\n type: {\n source: parquet\n sink: client\n path: \"hdfs://192.168.11.13:9000/data/edge_serve.parquet\"\n fields: [start_year,end_year]\n nebula.fields: [start_year, end_year]\n source: {\n field: src\n target: {\n field: dst\n batch: 256\n partition: 32\n # If more edges need to be added, refer to the previous configuration to add them.\nStep 4: Import data into NebulaGraph\nRun the following command to import Parquet data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/parquet_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 5: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 6: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-parquet/","type":"doc"},{"title":"Import data from HBase","content":"This topic provides an example of how to use Exchange to import NebulaGraph data stored in HBase.\nData set\nThis topic takes the basketballplayer dataset as an example.\nIn this example, the data set has been stored in HBase. All vertexes and edges are stored in the player, team, follow, and serve tables. The following are some of the data for each table.\nhbase(main):002:0> scan \"player\"\nROW COLUMN+CELL\n player100 column=cf:age, timestamp=1618881347530, value=42\n player100 column=cf:name, timestamp=1618881354604, value=Tim Duncan\n player101 column=cf:age, timestamp=1618881369124, value=36\n player101 column=cf:name, timestamp=1618881379102, value=Tony Parker\n player102 column=cf:age, timestamp=1618881386987, value=33\n player102 column=cf:name, timestamp=1618881393370, value=LaMarcus Aldridge\n player103 column=cf:age, timestamp=1618881402002, value=32\n player103 column=cf:name, timestamp=1618881407882, value=Rudy Gay\n ...\nhbase(main):003:0> scan \"team\"\nROW COLUMN+CELL\n team200 column=cf:name, timestamp=1618881445563, value=Warriors\n team201 column=cf:name, timestamp=1618881453636, value=Nuggets\n ...\nhbase(main):004:0> scan \"follow\"\nROW COLUMN+CELL\n player100 column=cf:degree, timestamp=1618881804853, value=95\n player100 column=cf:dst_player, timestamp=1618881791522, value=player101\n player101 column=cf:degree, timestamp=1618881824685, value=90\n player101 column=cf:dst_player, timestamp=1618881816042, value=player102\n ...\nhbase(main):005:0> scan \"serve\"\nROW COLUMN+CELL\n player100 column=cf:end_year, timestamp=1618881899333, value=2016\n player100 column=cf:start_year, timestamp=1618881890117, value=1997\n player100 column=cf:teamid, timestamp=1618881875739, value=team204\n ...\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nHBase: 2.2.7\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set HBase data source configuration. In this example, the copied file is called hbase_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and all Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set information about Tag player.\n # If you want to set RowKey as the data source, enter rowkey and the actual column name of the column family.\n # The Tag name in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to HBase.\n source: hbase\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n host:192.168.*.*\n port:2181\n table:\"player\"\n columnFamily:\"cf\"\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [age,name]\n nebula.fields: [age,name]\n # Specify a column of data in the table as the source of vertex VID in the NebulaGraph.\n # For example, if rowkey is the source of the VID, enter rowkey.\n vertex:{\n field:rowkey\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # Number of pieces of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set Tag Team information.\n name: team\n type: {\n source: hbase\n sink: client\n host:192.168.*.*\n port:2181\n table:\"team\"\n columnFamily:\"cf\"\n fields: [name]\n nebula.fields: [name]\n vertex:{\n field:rowkey\n batch: 256\n partition: 32\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # The corresponding Edge Type name in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to HBase.\n source: hbase\n # Specify how to import the Edge type data into NebulaGraph.\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n host:192.168.*.*\n port:2181\n table:\"follow\"\n columnFamily:\"cf\"\n # Specify the column names in the follow table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [degree]\n nebula.fields: [degree]\n # In source, use a column in the follow table as the source of the edge's source vertex.\n # In target, use a column in the follow table as the source of the edge's destination vertex.\n source:{\n field:rowkey\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target:{\n field:dst_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge Type serve.\n name: serve\n type: {\n source: hbase\n sink: client\n host:192.168.*.*\n port:2181\n table:\"serve\"\n columnFamily:\"cf\"\n fields: [start_year,end_year]\n nebula.fields: [start_year,end_year]\n source:{\n field:rowkey\n target:{\n field:teamid\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n batch: 256\n partition: 32\nStep 3: Import data into NebulaGraph\nRun the following command to import HBase data into NebulaGraph. For descriptions of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/hbase_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 4: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 5: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-hbase/","type":"doc"},{"title":"Import data from MySQL/PostgreSQL","content":"This topic provides an example of how to use Exchange to export MySQL data and import to NebulaGraph. It also applies to exporting\ndata from PostgreSQL into NebulaGraph.\nData set\nThis topic takes the basketballplayer dataset as an example.\nIn this example, the data set has been stored in MySQL. All vertexes and edges are stored in the player, team, follow, and serve tables. The following are some of the data for each table.\nmysql> desc player;\nmysql> desc team;\nmysql> desc follow;\nmysql> desc serve;\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nMySQL: 8.0.23\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nmysql-connector-java-xxx.jar has been downloaded and placed in the directory SPARK_HOME/jars of Spark.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nThe Hadoop service has been installed and started.\nPrecautions\nnebula-exchange_spark_2.2 supports only single table queries, not multi-table queries.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set MySQL data source configuration. In this case, the copied file is called mysql_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n # The Tag name in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to MySQL.\n source: mysql\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n host:192.168.*.*\n port:3306\n user:\"test\"\n password:\"123456\"\n database:\"basketball\"\n # Scanning a single table to read data.\n # nebula-exchange_spark_2.2 must configure this parameter. Sentence is not supported.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as sentence.\n table:\"basketball.player\"\n # Use query statement to read data.\n # This parameter is not supported by nebula-exchange_spark_2.2.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as table. Multi-table queries are supported.\n # sentence: \"select * from people, player, team\"\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [age,name]\n nebula.fields: [age,name]\n # Specify a column of data in the table as the source of VIDs in the NebulaGraph.\n vertex: {\n field:playerid\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag Team.\n name: team\n type: {\n source: mysql\n sink: client\n host:192.168.*.*\n port:3306\n database:\"basketball\"\n table:\"team\"\n user:\"test\"\n password:\"123456\"\n sentence:\"select teamid, name from team order by teamid;\"\n fields: [name]\n nebula.fields: [name]\n vertex: {\n field: teamid\n batch: 256\n partition: 32\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # The corresponding Edge Type name in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to MySQL.\n source: mysql\n # Specify how to import the Edge type data into NebulaGraph.\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n host:192.168.*.*\n port:3306\n user:\"test\"\n password:\"123456\"\n database:\"basketball\"\n # Scanning a single table to read data.\n # nebula-exchange_spark_2.2 must configure this parameter. Sentence is not supported.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as sentence.\n table:\"basketball.follow\"\n # Use query statement to read data.\n # This parameter is not supported by nebula-exchange_spark_2.2.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as table. Multi-table queries are supported.\n # sentence: \"select * from follow, serve\"\n # Specify the column names in the follow table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [degree]\n nebula.fields: [degree]\n # In source, use a column in the follow table as the source of the edge's source vertex.\n # In target, use a column in the follow table as the source of the edge's destination vertex.\n source: {\n field: src_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: dst_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge Type serve.\n name: serve\n type: {\n source: mysql\n sink: client\n host:192.168.*.*\n port:3306\n database:\"basketball\"\n table:\"serve\"\n user:\"test\"\n password:\"123456\"\n sentence:\"select playerid,teamid,start_year,end_year from serve order by playerid;\"\n fields: [start_year,end_year]\n nebula.fields: [start_year,end_year]\n source: {\n field: playerid\n target: {\n field: teamid\n batch: 256\n partition: 32\nStep 3: Import data into NebulaGraph\nRun the following command to import MySQL data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/mysql_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 4: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 5: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-mysql/","type":"doc"},{"title":"Import data from Oracle","content":"This topic provides an example of how to use Exchange to export Oracle data and import to NebulaGraph.\nData set\nThis topic takes the basketballplayer dataset as an example.\nIn this example, the data set has been stored in Oracle. All vertexes and edges are stored in the player, team, follow, and serve tables. The following are some of the data for each table.\noracle> desc player;\n+-----------+-------+---------------+ \n+-----------+-------+---------------+ \n+-----------+-------+---------------+ \noracle> desc team;\n+-----------+-------+---------------+ \n+-----------+-------+---------------+ \n+-----------+-------+---------------+ \noracle> desc follow;\n+-------------+-------+---------------+ \n+-------------+-------+---------------+ \n+-------------+-------+---------------+ \noracle> desc serve;\n+------------+-------+---------------+ \n+------------+-------+---------------+ \n+------------+-------+---------------+ \nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nThe Hadoop service has been installed and started.\nPrecautions\nnebula-exchange_spark_2.2 supports only single table queries, not multi-table queries.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set Oracle data source configuration. In this case, the copied file is called oracle_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n # The Tag name in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to Oracle.\n source: oracle\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n url:\"jdbc:oracle:thin:@host:1521:basketball\"\n driver: \"oracle.jdbc.driver.OracleDriver\"\n user: \"root\"\n password: \"123456\"\n # Scanning a single table to read data.\n # nebula-exchange_spark_2.2 must configure this parameter. Sentence is not supported.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as sentence.\n table:\"basketball.player\"\n # Use query statement to read data.\n # This parameter is not supported by nebula-exchange_spark_2.2.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as table. Multi-table queries are supported.\n # sentence: \"select * from people, player, team\"\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [age,name]\n nebula.fields: [age,name]\n # Specify a column of data in the table as the source of VIDs in the NebulaGraph.\n vertex: {\n field:playerid\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag Team.\n name: team\n type: {\n source: oracle\n sink: client\n url:\"jdbc:oracle:thin:@host:1521:basketball\"\n driver: \"oracle.jdbc.driver.OracleDriver\"\n user: \"root\"\n password: \"123456\"\n table: \"basketball.team\"\n sentence: \"select teamid, name from team\"\n fields: [name]\n nebula.fields: [name]\n vertex: {\n field: teamid\n batch: 256\n partition: 32\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # The corresponding Edge Type name in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to Oracle.\n source: oracle\n # Specify how to import the Edge type data into NebulaGraph.\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n url:\"jdbc:oracle:thin:@host:1521:basketball\"\n driver: \"oracle.jdbc.driver.OracleDriver\"\n user: \"root\"\n password: \"123456\"\n # Scanning a single table to read data.\n # nebula-exchange_spark_2.2 must configure this parameter. Sentence is not supported.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as sentence.\n table:\"basketball.follow\"\n # Use query statement to read data.\n # This parameter is not supported by nebula-exchange_spark_2.2.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as table. Multi-table queries are supported.\n # sentence: \"select * from follow, serve\"\n # Specify the column names in the follow table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [degree]\n nebula.fields: [degree]\n # In source, use a column in the follow table as the source of the edge's source vertex.\n # In target, use a column in the follow table as the source of the edge's destination vertex.\n source: {\n field: src_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: dst_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge Type serve.\n name: serve\n type: {\n source: oracle\n sink: client\n url:\"jdbc:oracle:thin:@host:1521:basketball\"\n driver: \"oracle.jdbc.driver.OracleDriver\"\n user: \"root\"\n password: \"123456\"\n table: \"basketball.serve\"\n sentence: \"select playerid, teamid, start_year, end_year from serve\"\n fields: [start_year,end_year]\n nebula.fields: [start_year,end_year]\n source: {\n field: playerid\n target: {\n field: teamid\n batch: 256\n partition: 32\nStep 3: Import data into NebulaGraph\nRun the following command to import Oracle data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/oracle_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 4: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 5: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-oracle/","type":"doc"},{"title":"Import data from ClickHouse","content":"This topic provides an example of how to use Exchange to import data stored on ClickHouse into NebulaGraph.\nData set\nThis topic takes the basketballplayer dataset as an example.\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nClickHouse: docker deployment yandex/clickhouse-server tag: latest(2021.07.01)\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set ClickHouse data source configuration. In this example, the copied file is called clickhouse_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n cores: {\n max: 16\n# NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n name: player\n type: {\n # Specify the data source file format to ClickHouse.\n source: clickhouse\n # Specify how to import the data of vertexes into NebulaGraph: Client or SST.\n sink: client\n # JDBC URL of ClickHouse\n url:\"jdbc:clickhouse://192.168.*.*:8123/basketballplayer\"\n user:\"user\"\n password:\"123456\"\n # The number of ClickHouse partitions\n numPartition:\"5\"\n sentence:\"select * from player\"\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [name,age]\n nebula.fields: [name,age]\n # Specify a column of data in the table as the source of vertex VID in the NebulaGraph.\n vertex: {\n field:playerid\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag Team.\n name: team\n type: {\n source: clickhouse\n sink: client\n url:\"jdbc:clickhouse://192.168.*.*:8123/basketballplayer\"\n user:\"user\"\n password:\"123456\"\n numPartition:\"5\"\n sentence:\"select * from team\"\n fields: [name]\n nebula.fields: [name]\n vertex: {\n field:teamid\n batch: 256\n partition: 32\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # The corresponding Edge Type name in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to ClickHouse.\n source: clickhouse\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # JDBC URL of ClickHouse\n url:\"jdbc:clickhouse://192.168.*.*:8123/basketballplayer\"\n user:\"user\"\n password:\"123456\"\n # The number of ClickHouse partitions.\n numPartition:\"5\"\n sentence:\"select * from follow\"\n # Specify the column names in the follow table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [degree]\n nebula.fields: [degree]\n # In source, use a column in the follow table as the source of the edge's source vertexes.\n source: {\n field:src_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # In target, use a column in the follow table as the source of the edge's destination vertexes.\n target: {\n field:dst_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge Type serve.\n name: serve\n type: {\n source: clickhouse\n sink: client\n url:\"jdbc:clickhouse://192.168.*.*:8123/basketballplayer\"\n user:\"user\"\n password:\"123456\"\n numPartition:\"5\"\n sentence:\"select * from serve\"\n fields: [start_year,end_year]\n nebula.fields: [start_year,end_year]\n source: {\n field:playerid\n target: {\n field:teamid\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n batch: 256\n partition: 32\nStep 3: Import data into NebulaGraph\nRun the following command to import ClickHouse data into NebulaGraph. For descriptions of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/clickhouse_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 4: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 5: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-clickhouse/","type":"doc"},{"title":"Import data from Neo4j","content":"This topic provides an example of how to use Exchange to import NebulaGraph data stored in Neo4j.\nImplementation method\nExchange uses Neo4j Driver 4.0.1 to read Neo4j data. Before batch export, you need to write Cypher statements that are automatically executed based on labels and relationship types and the number of Spark partitions in the configuration file to improve data export performance.\nWhen Exchange reads Neo4j data, it needs to do the following:\nThe Reader in Exchange replaces the statement following the Cypher RETURN statement in the exec part of the configuration file with COUNT(*), and executes this statement to get the total amount of data, then calculates the starting offset and size of each partition based on the number of Spark partitions.\n(Optional) If the user has configured the check_point_path directory, Reader reads the files in the directory. In the transferring state, Reader calculates the offset and size that each Spark partition should have.\nIn each Spark partition, the Reader in Exchange adds different SKIP and LIMIT statements to the Cypher statement and calls the Neo4j Driver for parallel execution to distribute data to different Spark partitions.\nThe Reader finally processes the returned data into a DataFrame.\nAt this point, Exchange has finished exporting the Neo4j data. The data is then written in parallel to the NebulaGraph database.\nThe whole process is illustrated below.\nData set\nThis topic takes the basketballplayer dataset as an example.\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU：Intel(R) Xeon(R) CPU E5-2697 v3 @ 2.60GHz\nCPU cores: 14\nMemory: 251 GB\nSpark: Stand-alone, 2.4.6 pre-build for Hadoop 2.7\nNeo4j: 3.5.20 Community Edition\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with NebulaGraph write permission.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer\n USE basketballplayer;\n## Create the Tag player\n CREATE TAG player(name string, age int);\n## Create the Tag team\n CREATE TAG team(name string);\n## Create the Edge type follow\n CREATE EDGE follow(degree int);\n## Create the Edge type serve\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Configuring source data\nTo speed up the export of Neo4j data, create indexes for the corresponding properties in the Neo4j database. For more information, refer to the Neo4j manual.\nStep 3: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set Neo4j data source configuration. In this example, the copied file is called neo4j_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n executor: {\n memory:1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n user: root\n pswd: nebula\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player\n name: player\n type: {\n source: neo4j\n sink: client\n server: \"bolt://192.168.*.*:7687\"\n user: neo4j\n password:neo4j\n # bolt 3 does not support multiple databases, do not configure database names. 4 and above can configure database names.\n # database:neo4j\n exec: \"match (n:player) return n.id as id, n.age as age, n.name as name\"\n fields: [age,name]\n nebula.fields: [age,name]\n vertex: {\n field:id\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n partition: 10\n batch: 1000\n check_point_path: /tmp/test\n # Set the information about the Tag Team\n name: team\n type: {\n source: neo4j\n sink: client\n server: \"bolt://192.168.*.*:7687\"\n user: neo4j\n password:neo4j\n database:neo4j\n exec: \"match (n:team) return n.id as id,n.name as name\"\n fields: [name]\n nebula.fields: [name]\n vertex: {\n field:id\n partition: 10\n batch: 1000\n check_point_path: /tmp/test\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow\n name: follow\n type: {\n source: neo4j\n sink: client\n server: \"bolt://192.168.*.*:7687\"\n user: neo4j\n password:neo4j\n # bolt 3 does not support multiple databases, do not configure database names. 4 and above can configure database names.\n # database:neo4j\n exec: \"match (a:player)-[r:follow]->(b:player) return a.id as src, b.id as dst, r.degree as degree order by id(r)\"\n fields: [degree]\n nebula.fields: [degree]\n source: {\n field: src\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: dst\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n partition: 10\n batch: 1000\n check_point_path: /tmp/test\n # Set the information about the Edge Type serve\n name: serve\n type: {\n source: neo4j\n sink: client\n server: \"bolt://192.168.*.*:7687\"\n user: neo4j\n password:neo4j\n database:neo4j\n exec: \"match (a:player)-[r:serve]->(b:team) return a.id as src, b.id as dst, r.start_year as start_year, r.end_year as end_year order by id(r)\"\n fields: [start_year,end_year]\n nebula.fields: [start_year,end_year]\n source: {\n field: src\n target: {\n field: dst\n #ranking: rank\n partition: 10\n batch: 1000\n check_point_path: /tmp/test\nExec configuration\nWhen configuring either the tags.exec or edges.exec parameters, you need to fill in the Cypher query. To prevent loss of data during import, it is strongly recommended to include ORDER BY clause in Cypher queries. Meanwhile, in order to improve data import efficiency, it is better to select indexed properties for ordering. If there is no index, users can also observe the default order and select the appropriate properties for ordering to improve efficiency. If the pattern of the default order cannot be found, users can order them by the ID of the vertex or relationship and set the partition to a small value to reduce the ordering pressure of Neo4j.\nExchange needs to execute different SKIP and LIMIT Cypher statements on different Spark partitions, so SKIP and LIMIT clauses cannot be included in the Cypher statements corresponding to tags.exec and edges.exec.\ntags.vertex or edges.vertex configuration\nNebulaGraph uses ID as the unique primary key when creating vertexes and edges, overwriting the data in that primary key if it already exists. So, if a Neo4j property value is given as the NebulaGraph'S ID and the value is duplicated in Neo4j, duplicate IDs will be generated. One and only one of their corresponding data will be stored in the NebulaGraph, and the others will be overwritten. Because the data import process is concurrently writing data to NebulaGraph, the final saved data is not guaranteed to be the latest data in Neo4j.\ncheck_point_path configuration\nIf breakpoint transfers are enabled, to avoid data loss, the state of the database should not change between the breakpoint and the transfer. For example, data cannot be added or deleted, and the partition quantity configuration should not be changed.\nStep 4: Import data into NebulaGraph\nRun the following command to import Neo4j data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/neo4j_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 5: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 6: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-neo4j/","type":"doc"},{"title":"Import data from Hive","content":"This topic provides an example of how to use Exchange to import NebulaGraph data stored in Hive.\nData set\nThis topic takes the basketballplayer dataset as an example.\nIn this example, the data set has been stored in Hive. All vertexes and edges are stored in the player, team, follow, and serve tables. The following are some of the data for each table.\nscala> spark.sql(\"describe basketball.player\").show\nscala> spark.sql(\"describe basketball.team\").show\nscala> spark.sql(\"describe basketball.follow\").show\nscala> spark.sql(\"describe basketball.serve\").show\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nHadoop: 2.9.2, pseudo-distributed deployment\nHive: 2.3.7, Hive Metastore database is MySQL 8.0.22\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nThe Hive Metastore database (MySQL in this example) has been started.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer\n USE basketballplayer;\n## Create the Tag player\n CREATE TAG player(name string, age int);\n## Create the Tag team\n CREATE TAG team(name string);\n## Create the Edge type follow\n CREATE EDGE follow(degree int);\n## Create the Edge type serve\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Use Spark SQL to confirm Hive SQL statements\nAfter the Spark-shell environment is started, run the following statements to ensure that Spark can read data in Hive.\nscala> sql(\"select playerid, age, name from basketball.player\").show\nscala> sql(\"select teamid, name from basketball.team\").show\nscala> sql(\"select src_player, dst_player, degree from basketball.follow\").show\nscala> sql(\"select playerid, teamid, start_year, end_year from basketball.serve\").show\nThe following is the result read from the table basketball.player.\n...\nStep 3: Modify configuration file\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set Hive data source configuration. In this example, the copied file is called hive_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n cores: {\n max: 16\n # If Spark and Hive are deployed in different clusters, you need to configure the parameters for connecting to Hive. Otherwise, skip these configurations.\n #hive: {\n # waredir: \"hdfs://NAMENODE_IP:9000/apps/svr/hive-xxx/warehouse/\"\n # connectionURL: \"jdbc:mysql://your_ip:3306/hive_spark?characterEncoding=UTF-8\"\n # connectionDriverName: \"com.mysql.jdbc.Driver\"\n # connectionUserName: \"user\"\n # connectionPassword: \"password\"\n #}\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and all Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n # The Tag name in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to Hive.\n source: hive\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Set the SQL statement to read the data of player table in basketball database.\n exec: \"select playerid, age, name from basketball.player\"\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [age,name]\n nebula.fields: [age,name]\n # Specify a column of data in the table as the source of vertex VID in the NebulaGraph.\n vertex:{\n field:playerid\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag Team.\n name: team\n type: {\n source: hive\n sink: client\n exec: \"select teamid, name from basketball.team\"\n fields: [name]\n nebula.fields: [name]\n vertex: {\n field: teamid\n batch: 256\n partition: 32\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # The corresponding Edge Type name in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to Hive.\n source: hive\n # Specify how to import the Edge type data into NebulaGraph.\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Set the SQL statement to read the data of follow table in the basketball database.\n exec: \"select src_player, dst_player, degree from basketball.follow\"\n # Specify the column names in the follow table in Fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [degree]\n nebula.fields: [degree]\n # In source, use a column in the follow table as the source of the edge's starting vertex.\n # In target, use a column in the follow table as the source of the edge's destination vertex.\n source: {\n field: src_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: dst_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge Type serve.\n name: serve\n type: {\n source: hive\n sink: client\n exec: \"select playerid, teamid, start_year, end_year from basketball.serve\"\n fields: [start_year,end_year]\n nebula.fields: [start_year,end_year]\n source: {\n field: playerid\n target: {\n field: teamid\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n batch: 256\n partition: 32\nStep 4: Import data into NebulaGraph\nRun the following command to import Hive data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c -h\nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/hive_application.conf -h\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 5: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 6: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-hive/","type":"doc"},{"title":"Import data from MaxCompute","content":"This topic provides an example of how to use Exchange to import NebulaGraph data stored in MaxCompute.\nData set\nThis topic takes the basketballplayer dataset as an example.\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nMaxCompute: Alibaba Cloud official version\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set MaxCompute data source configuration. In this example, the copied file is called maxcompute_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n name: player\n type: {\n # Specify the data source file format to MaxCompute.\n source: maxcompute\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # Table name of MaxCompute.\n table:player\n # Project name of MaxCompute.\n project:project\n # OdpsUrl and tunnelUrl for the MaxCompute service.\n # The address is https://help.aliyun.com/document_detail/34951.html.\n odpsUrl:\"http://service.cn-hangzhou.maxcompute.aliyun.com/api\"\n tunnelUrl:\"http://dt.cn-hangzhou.maxcompute.aliyun.com\"\n # AccessKeyId and accessKeySecret of the MaxCompute service.\n accessKeyId:xxx\n accessKeySecret:xxx\n # Partition description of the MaxCompute table. This configuration is optional.\n partitionSpec:\"dt='partition1'\"\n # Ensure that the table name in the SQL statement is the same as the value of the table above. This configuration is optional.\n sentence:\"select id, name, age, playerid from player where id < 10\"\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields:[name, age]\n nebula.fields:[name, age]\n # Specify a column of data in the table as the source of vertex VID in the NebulaGraph.\n vertex:{\n field: playerid\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag Team.\n name: team\n type: {\n source: maxcompute\n sink: client\n table:team\n project:project\n odpsUrl:\"http://service.cn-hangzhou.maxcompute.aliyun.com/api\"\n tunnelUrl:\"http://dt.cn-hangzhou.maxcompute.aliyun.com\"\n accessKeyId:xxx\n accessKeySecret:xxx\n partitionSpec:\"dt='partition1'\"\n sentence:\"select id, name, teamid from team where id < 10\"\n fields:[name]\n nebula.fields:[name]\n vertex:{\n field: teamid\n batch: 256\n partition: 32\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # The corresponding Edge Type name in NebulaGraph.\n name: follow\n type:{\n # Specify the data source file format to MaxCompute.\n source:maxcompute\n # Specify how to import the Edge type data into NebulaGraph.\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink:client\n # Table name of MaxCompute.\n table:follow\n # Project name of MaxCompute.\n project:project\n # OdpsUrl and tunnelUrl for MaxCompute service.\n # The address is https://help.aliyun.com/document_detail/34951.html.\n odpsUrl:\"http://service.cn-hangzhou.maxcompute.aliyun.com/api\"\n tunnelUrl:\"http://dt.cn-hangzhou.maxcompute.aliyun.com\"\n # AccessKeyId and accessKeySecret of the MaxCompute service.\n accessKeyId:xxx\n accessKeySecret:xxx\n # Partition description of the MaxCompute table. This configuration is optional.\n partitionSpec:\"dt='partition1'\"\n # Ensure that the table name in the SQL statement is the same as the value of the table above. This configuration is optional.\n sentence:\"select * from follow\"\n # Specify the column names in the follow table in Fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields:[degree]\n nebula.fields:[degree]\n # In source, use a column in the follow table as the source of the edge's source vertex.\n source:{\n field: src_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # In target, use a column in the follow table as the source of the edge's destination vertex.\n target:{\n field: dst_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition:10\n # The number of data written to NebulaGraph in a single batch.\n batch:10\n # Set the information about the Edge Type serve.\n name: serve\n type:{\n source:maxcompute\n sink:client\n table:serve\n project:project\n odpsUrl:\"http://service.cn-hangzhou.maxcompute.aliyun.com/api\"\n tunnelUrl:\"http://dt.cn-hangzhou.maxcompute.aliyun.com\"\n accessKeyId:xxx\n accessKeySecret:xxx\n partitionSpec:\"dt='partition1'\"\n sentence:\"select * from serve\"\n fields:[start_year,end_year]\n nebula.fields:[start_year,end_year]\n source:{\n field: playerid\n target:{\n field: teamid\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n partition:10\n batch:10\nStep 3: Import data into NebulaGraph\nRun the following command to import MaxCompute data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/maxcompute_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 4: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 5: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-maxcompute/","type":"doc"},{"title":"Import data from Pulsar","content":"This topic provides an example of how to use Exchange to import NebulaGraph data stored in Pulsar.\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nThe Pulsar service has been installed and started.\nPrecautions\nOnly client mode is supported when importing Pulsar data, i.e. the value of parameters tags.type.sink and edges.type.sink is client.\nWhen importing Pulsar data, do not use Exchange version 3.4.0, which adds caching of imported data and does not support streaming data import. Use Exchange versions 3.0.0, 3.3.0, or 3.5.0.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer\n USE basketballplayer;\n## Create the Tag player\n CREATE TAG player(name string, age int);\n## Create the Tag team\n CREATE TAG team(name string);\n## Create the Edge type follow\n CREATE EDGE follow(degree int);\n## Create the Edge type serve\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set Pulsar data source configuration. In this example, the copied file is called pulsar_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and all Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertices\n tags: [\n # Set the information about the Tag player.\n # The corresponding Tag name in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to Pulsar.\n source: pulsar\n # Specify how to import the data into NebulaGraph. Only client is supported.\n sink: client\n # The address of the Pulsar server.\n service: \"pulsar://127.0.0.1:6650\"\n # admin.url of pulsar.\n admin: \"http://127.0.0.1:8081\"\n # The Pulsar option can be configured from topic, topics or topicsPattern.\n options: {\n topics: \"topic1,topic2\"\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [age,name]\n nebula.fields: [age,name]\n # Specify a column of data in the table as the source of VIDs in the NebulaGraph.\n vertex:{\n field:playerid\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 10\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 10\n # The interval for message reading. Unit: second.\n interval.seconds: 10\n # Set the information about the Tag Team.\n name: team\n type: {\n source: pulsar\n sink: client\n service: \"pulsar://127.0.0.1:6650\"\n admin: \"http://127.0.0.1:8081\"\n options: {\n topics: \"topic1,topic2\"\n fields: [name]\n nebula.fields: [name]\n vertex:{\n field:teamid\n batch: 10\n partition: 10\n interval.seconds: 10\n # Processing edges\n edges: [\n # Set the information about Edge Type follow\n # The corresponding Edge Type name in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to Pulsar.\n source: pulsar\n # Specify how to import the Edge type data into NebulaGraph.\n # Specify how to import the data into NebulaGraph. Only client is supported.\n sink: client\n # The address of the Pulsar server.\n service: \"pulsar://127.0.0.1:6650\"\n # admin.url of pulsar.\n admin: \"http://127.0.0.1:8081\"\n # The Pulsar option can be configured from topic, topics or topicsPattern.\n options: {\n topics: \"topic1,topic2\"\n # Specify the column names in the follow table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [degree]\n nebula.fields: [degree]\n # In source, use a column in the follow table as the source of the edge's source vertex.\n # In target, use a column in the follow table as the source of the edge's destination vertex.\n source:{\n field:src_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target:{\n field:dst_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 10\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 10\n # The interval for message reading. Unit: second.\n interval.seconds: 10\n # Set the information about the Edge Type serve\n name: serve\n type: {\n source: Pulsar\n sink: client\n service: \"pulsar://127.0.0.1:6650\"\n admin: \"http://127.0.0.1:8081\"\n options: {\n topics: \"topic1,topic2\"\n fields: [start_year,end_year]\n nebula.fields: [start_year,end_year]\n source:{\n field:playerid\n target:{\n field:teamid\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n batch: 10\n partition: 10\n interval.seconds: 10\nStep 3: Import data into NebulaGraph\nRun the following command to import Pulsar data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/pulsar_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 4: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 5: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-pulsar/","type":"doc"},{"title":"Import data from Kafka","content":"This topic provides a simple guide to importing Data stored on Kafka into NebulaGraph using Exchange.\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nThe following JAR files have been downloaded and placed in the directory SPARK_HOME/jars of Spark:\nspark-streaming-kafka_xxx.jar\nspark-sql-kafka-0-10_xxx.jar\nkafka-clients-xxx.jar\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nThe Kafka service has been installed and started.\nPrecautions\nOnly client mode is supported when importing Kafka data, i.e. the value of parameters tags.type.sink and edges.type.sink is client.\nWhen importing Kafka data, do not use Exchange version 3.4.0, which adds caching of imported data and does not support streaming data import. Use Exchange versions 3.0.0, 3.3.0, or 3.5.0.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set Kafka data source configuration. In this example, the copied file is called kafka_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and all Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n # The corresponding Tag name in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to Kafka.\n source: kafka\n # Specify how to import the data into NebulaGraph. Only client is supported.\n sink: client\n # Kafka server address.\n service: \"127.0.0.1:9092\"\n # Message category.\n topic: \"topic_name1\"\n # Kafka data has a fixed domain name: key, value, topic, partition, offset, timestamp, timestampType.\n # If multiple fields need to be specified after Spark reads as DataFrame, separate them with commas.\n # Specify the field name in fields. For example, use key for name in NebulaGraph and value for age in Nebula, as shown in the following.\n fields: [key,value]\n nebula.fields: [name,age]\n # Specify a column of data in the table as the source of vertex VID in the NebulaGraph.\n # The key is the same as the value above, indicating that key is used as both VID and property name.\n vertex:{\n field:key\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 10\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 10\n # The interval for message reading. Unit: second.\n interval.seconds: 10\n # The consumer offsets. The default value is latest. Optional value are latest and earliest.\n startingOffsets: latest\n # Flow control, with a rate limit on the maximum offset processed per trigger interval, may not be configured.\n # maxOffsetsPerTrigger:10000\n # Processing edges\n #edges: [\n # # Set the information about the Edge Type follow.\n # {\n # # The corresponding Edge Type name in NebulaGraph.\n # name: follow\n # type: {\n # # Specify the data source file format to Kafka.\n # source: kafka\n # # Specify how to import the Edge type data into NebulaGraph.\n # # Specify how to import the data into NebulaGraph. Only client is supported.\n # sink: client\n # }\n # # Kafka server address.\n # service: \"127.0.0.1:9092\"\n # # Message category.\n # topic: \"topic_name3\"\n # # Kafka data has a fixed domain name: key, value, topic, partition, offset, timestamp, timestampType.\n # # If multiple fields need to be specified after Spark reads as DataFrame, separate them with commas.\n # # Specify the field name in fields. For example, use key for degree in Nebula, as shown in the following.\n # fields: [key]\n # nebula.fields: [degree]\n # # In source, use a column in the topic as the source of the edge's source vertex.\n # # In target, use a column in the topic as the source of the edge's destination vertex.\n # source:{\n # field:timestamp\n # # udf:{\n # # separator:\"_\"\n # # oldColNames:[field-0,field-1,field-2]\n # # newColName:new-field\n # # }\n # # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # # prefix:\"tag1\"\n # # Performs hashing operations on VIDs of type string.\n # # policy:hash\n # }\n # target:{\n # field:offset\n # # udf:{\n # # separator:\"_\"\n # # oldColNames:[field-0,field-1,field-2]\n # # newColName:new-field\n # # }\n # # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # # prefix:\"tag1\"\n # # Performs hashing operations on VIDs of type string.\n # # policy:hash\n # }\n # # (Optional) Specify a column as the source of the rank.\n # #ranking: rank\n # # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n # #writeMode: INSERT\n # # The number of data written to NebulaGraph in a single batch.\n # batch: 10\n # # The number of partitions to be created when the data is written to NebulaGraph.\n # partition: 10\n # # The interval for message reading. Unit: second.\n # interval.seconds: 10\n # # The consumer offsets. The default value is latest. Optional value are latest and earliest.\n # startingOffsets: latest\n # # Flow control, with a rate limit on the maximum offset processed per trigger interval, may not be configured.\n # # maxOffsetsPerTrigger:10000\n # }\n #]\nStep 3: Import data into NebulaGraph\nRun the following command to import Kafka data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/kafka_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 4: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 5: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-kafka/","type":"doc"},{"title":"Import data from general JDBC","content":"JDBC data refers to the data of various databases accessed through the JDBC interface. This topic provides an example of how to use Exchange to export MySQL data and import to NebulaGraph.\nData set\nThis topic takes the basketballplayer dataset as an example.\nIn this example, the data set has been stored in MySQL. All vertexes and edges are stored in the player, team, follow, and serve tables. The following are some of the data for each table.\nmysql> desc player;\nmysql> desc team;\nmysql> desc follow;\nmysql> desc serve;\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nMySQL: 8.0.23\nNebulaGraph: 3.6.0. Deploy NebulaGraph with Docker Compose.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nLearn about the Schema created in NebulaGraph, including names and properties of Tags and Edge types, and more.\nThe Hadoop service has been installed and started.\nPrecautions\nnebula-exchange_spark_2.2 supports only single table queries, not multi-table queries.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space.\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer.\n USE basketballplayer;\n## Create the Tag player.\n CREATE TAG player(name string, age int);\n## Create the Tag team.\n CREATE TAG team(name string);\n## Create the Edge type follow.\n CREATE EDGE follow(degree int);\n## Create the Edge type serve.\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set JDBC data source configuration. In this case, the copied file is called jdbc_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n driver: {\n cores: 1\n maxResultSize: 1G\n cores: {\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n # Specify the IP addresses and ports for Graph and Meta services.\n # If there are multiple addresses, the format is \"ip1:port\",\"ip2:port\",\"ip3:port\".\n # Addresses are separated by commas.\n graph:[\"127.0.0.1:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"127.0.0.1:9559\"]\n # The account entered must have write permission for the NebulaGraph space.\n user: root\n pswd: nebula\n # Fill in the name of the graph space you want to write data to in the NebulaGraph.\n space: basketballplayer\n connection: {\n timeout: 3000\n retry: 3\n execution: {\n retry: 3\n error: {\n max: 32\n output: /tmp/errors\n rate: {\n limit: 1024\n timeout: 1000\n # Processing vertexes\n tags: [\n # Set the information about the Tag player.\n # The Tag name in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to JDBC.\n source: jdbc\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n # URL of the JDBC data source. The example is MySql database.\n url:\"jdbc:mysql://127.0.0.1:3306/basketball?useUnicode=true&characterEncoding=utf-8\"\n # JDBC driver \n driver:\"com.mysql.cj.jdbc.Driver\"\n # Database user name and password\n user:\"root\"\n password:\"12345\"\n # Scanning a single table to read data.\n # nebula-exchange_spark_2.2 must configure this parameter, and can additionally configure sentence.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as sentence.\n table:\"basketball.player\"\n # Use query statement to read data.\n # nebula-exchange_spark_2.2 can configure this parameter. Multi-table queries are not supported. Only the table name needs to be written after from. The form `db.table` is not supported.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as table. Multi-table queries are supported.\n # sentence:\"select playerid, age, name from player, team order by playerid\"\n # (optional)Multiple connections read parameters. See https://spark.apache.org/docs/latest/sql-data-sources-jdbc.html\n partitionColumn:playerid # optional. Must be a numeric, date, or timestamp column from the table in question.\n lowerBound:1 # optional\n upperBound:5 # optional\n numPartitions:5 # optional\n fetchSize:2 # The JDBC fetch size, which determines how many rows to fetch per round trip.\n # Specify the column names in the player table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [age,name]\n nebula.fields: [age,name]\n # Specify a column of data in the table as the source of VIDs in the NebulaGraph.\n vertex: {\n field:playerid\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.\n #deleteEdge: false\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Tag Team.\n name: team\n type: {\n source: jdbc\n sink: client\n url:\"jdbc:mysql://127.0.0.1:3306/basketball?useUnicode=true&characterEncoding=utf-8\"\n driver:\"com.mysql.cj.jdbc.Driver\"\n user:root\n password:\"12345\"\n table:team\n sentence:\"select teamid, name from team order by teamid\"\n partitionColumn:teamid \n lowerBound:1 \n upperBound:5 \n numPartitions:5 \n fetchSize:2 \n fields: [name]\n nebula.fields: [name]\n vertex: {\n field: teamid\n batch: 256\n partition: 32\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # The corresponding Edge Type name in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to JDBC.\n source: jdbc\n # Specify how to import the Edge type data into NebulaGraph.\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: client\n url:\"jdbc:mysql://127.0.0.1:3306/basketball?useUnicode=true&characterEncoding=utf-8\"\n driver:\"com.mysql.cj.jdbc.Driver\"\n user:root\n password:\"12345\"\n # Scanning a single table to read data.\n # nebula-exchange_spark_2.2 must configure this parameter, and can additionally configure sentence.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as sentence.\n table:\"basketball.follow\"\n # Use query statement to read data.\n # nebula-exchange_spark_2.2 can configure this parameter. Multi-table queries are not supported. Only the table name needs to be written after from. The form `db.table` is not supported.\n # nebula-exchange_spark_2.4 and nebula-exchange_spark_3.0 can configure this parameter, but not at the same time as table. Multi-table queries are supported.\n # sentence:\"select src_player,dst_player,degree from follow order by src_player\"\n partitionColumn:src_player \n lowerBound:1 \n upperBound:5 \n numPartitions:5 \n fetchSize:2 \n # Specify the column names in the follow table in fields, and their corresponding values are specified as properties in the NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n # If multiple column names need to be specified, separate them by commas.\n fields: [degree]\n nebula.fields: [degree]\n # In source, use a column in the follow table as the source of the edge's source vertex.\n # In target, use a column in the follow table as the source of the edge's destination vertex.\n source: {\n field: src_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: dst_player\n # udf:{\n # separator:\"_\"\n # oldColNames:[field-0,field-1,field-2]\n # newColName:new-field\n # }\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Set the information about the Edge Type serve.\n name: serve\n type: {\n source: jdbc\n sink: client\n url:\"jdbc:mysql://127.0.0.1:3306/basketball?useUnicode=true&characterEncoding=utf-8\"\n driver:\"com.mysql.cj.jdbc.Driver\"\n user:root\n password:\"12345\"\n table:serve\n sentence:\"select playerid,teamid,start_year,end_year from serve order by playerid\"\n partitionColumn:playerid \n lowerBound:1 \n upperBound:5 \n numPartitions:5 \n fetchSize:2\n fields: [start_year,end_year]\n nebula.fields: [start_year,end_year]\n source: {\n field: playerid\n target: {\n field: teamid\n batch: 256\n partition: 32\nStep 3: Import data into NebulaGraph\nRun the following command to import general JDBC data into NebulaGraph. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/jdbc_application.conf\nYou can search for batchSuccess. in the command output to check the number of successes. For example, batchSuccess.follow: 300.\nStep 4: (optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 5: (optional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-jdbc/","type":"doc"},{"title":"Import data from SST files","content":"This topic provides an example of how to generate the data from the data source into an SST (Sorted String Table) file and save it on HDFS, and then import it into NebulaGraph. The sample data source is a CSV file.\nPrecautions\nThe SST file can be imported only in Linux.\nThe default value of the property is not supported.\nBackground information\nExchange supports two data import modes:\nImport the data from the data source directly into NebulaGraph as nGQL statements.\nGenerate the SST file from the data source, and use Console to import the SST file into NebulaGraph.\nThe following describes the scenarios, implementation methods, prerequisites, and steps for generating an SST file and importing data.\nScenarios\nSuitable for online services, because the generation almost does not affect services (just reads the Schema), and the import speed is fast.\nSuitable for scenarios with a large amount of data from data sources for its fast import speed.\nImplementation methods\nThe underlying code in NebulaGraph uses RocksDB as the key-value storage engine. RocksDB is a storage engine based on the hard disk, providing a series of APIs for creating and importing SST files to help quickly import massive data.\nThe SST file is an internal file containing an arbitrarily long set of ordered key-value pairs for efficient storage of large amounts of key-value data. The entire process of generating SST files is mainly done by Exchange Reader, sstProcessor, and sstWriter. The whole data processing steps are as follows:\nReader reads data from the data source.\nsstProcessor generates the SST file from the NebulaGraph's Schema information and uploads it to the HDFS. For details about the format of the SST file, see Data Storage Format.\nsstWriter opens a file and inserts data. When generating SST files, keys must be written in sequence.\nAfter the SST file is generated, RocksDB imports the SST file into NebulaGraph using the IngestExternalFile() method. For example:\nIngestExternalFileOptions ifo;\n# Import two SST files\nStatus s = db_->IngestExternalFile({\"/home/usr/file1.sst\", \"/home/usr/file2.sst\"}, ifo);\nif (!s.ok()) {\n printf(\"Error while adding file %s and %s, Error %s\\n\",\n file_path1.c_str(), file_path2.c_str(), s.ToString().c_str());\n return 1;\nWhen the IngestExternalFile() method is called, RocksDB copies the file to the data directory by default and blocks the RocksDB write operation. If the key range in the SST file overwrites the Memtable key range, flush the Memtable to the hard disk. After placing the SST file in an optimal location in the LSM tree, assign a global serial number to the file and turn on the write operation.\nData set\nThis topic takes the basketballplayer dataset as an example.\nEnvironment\nThis example is done on MacOS. Here is the environment configuration information:\nHardware specifications:\nCPU: 1.7 GHz Quad-Core Intel Core i7\nMemory: 16 GB\nSpark: 2.4.7, stand-alone\nHadoop: 2.9.2, pseudo-distributed deployment\nNebulaGraph: 3.6.0.\nPrerequisites\nBefore importing data, you need to confirm the following information:\nNebulaGraph has been installed and deployed with the following information:\nIP addresses and ports of Graph and Meta services.\nThe user name and password with write permission to NebulaGraph.\n--ws_storage_http_port in the Meta service configuration file is the same as --ws_http_port in the Storage service configuration file. For example, 19779.\n--ws_meta_http_port in the Graph service configuration file is the same as --ws_http_port in the Meta service configuration file. For example, 19559.\nThe information about the Schema, including names and properties of Tags and Edge types, and more.\nExchange has been compiled, or download the compiled .jar file directly.\nSpark has been installed.\nJDK 1.8 or the later version has been installed and the environment variable JAVA_HOME has been configured.\nThe Hadoop service has been installed and started.\nSteps\nStep 1: Create the Schema in NebulaGraph\nAnalyze the data to create a Schema in NebulaGraph by following these steps:\nIdentify the Schema elements. The Schema elements in the NebulaGraph are shown in the following table.\nElement\nName\nProperty\nTag\nplayer\nname string, age int\nTag\nteam\nname string\nEdge Type\nfollow\ndegree int\nEdge Type\nserve\nstart_year int, end_year int\nCreate a graph space basketballplayer in the NebulaGraph and create a Schema as shown below.\n## Create a graph space\n CREATE SPACE basketballplayer \\\n (partition_num = 10, \\\n replica_factor = 1, \\\n vid_type = FIXED_STRING(30));\n## Use the graph space basketballplayer\n USE basketballplayer;\n## Create the Tag player\n CREATE TAG player(name string, age int);\n## Create the Tag team\n CREATE TAG team(name string);\n## Create the Edge type follow\n CREATE EDGE follow(degree int);\n## Create the Edge type serve\n CREATE EDGE serve(start_year int, end_year int);\nFor more information, see Quick start workflow.\nStep 2: Process CSV files\nConfirm the following information:\nProcess CSV files to meet Schema requirements.\nObtain the CSV file storage path.\nStep 3: Modify configuration files\nAfter Exchange is compiled, copy the conf file target/classes/application.conf to set SST data source configuration. In this example, the copied file is called sst_application.conf. For details on each configuration item, see Parameters in the configuration file.\n # Spark configuration\n spark: {\n app: {\n name: NebulaGraph Exchange 3.6.0\n master:local\n driver: {\n cores: 1\n maxResultSize: 1G\n executor: {\n memory:1G\n cores:{\n max: 16\n # NebulaGraph configuration\n nebula: {\n address:{\n graph:[\"192.8.168.XXX:9669\"]\n # the address of any of the meta services.\n # if your NebulaGraph server is in virtual network like k8s, please config the leader address of meta.\n meta:[\"192.8.168.XXX:9559\"]\n user: root\n pswd: nebula\n space: basketballplayer\n # SST file configuration\n path:{\n # The local directory that temporarily stores generated SST files\n local:\"/tmp\"\n # The path for storing the SST file in the HDFS\n remote:\"/sst\"\n # The NameNode address of HDFS, for example, \"hdfs://:\"\n hdfs.namenode: \"hdfs://*.*.*.*:9000\"\n # The connection parameters of clients\n connection: {\n # The timeout duration of socket connection and execution. Unit: milliseconds.\n timeout: 30000\n error: {\n # The maximum number of failures that will exit the application.\n max: 32\n # Failed import jobs are logged in the output path.\n output: /tmp/errors\n # Use Google's RateLimiter to limit requests to NebulaGraph.\n rate: {\n # Steady throughput of RateLimiter.\n limit: 1024\n # Get the allowed timeout duration from RateLimiter. Unit: milliseconds.\n timeout: 1000\n # Processing vertices\n tags: [\n # Set the information about the Tag player.\n # Specify the Tag name defined in NebulaGraph.\n name: player\n type: {\n # Specify the data source file format to CSV.\n source: csv\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: sst\n # Specify the path to the CSV file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example, \"hdfs://:port/xx/xx.csv\".\n path: \"hdfs://*.*.*.*:9000/dataset/vertex_player.csv\"\n # If the CSV file does not have a header, use [_c0, _c1, _c2, ..., _cn] to represent its header and indicate the columns as the source of the property values.\n # If the CSV file has a header, use the actual column name.\n fields: [_c1, _c2]\n # Specify the property name defined in NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [age, name]\n # Specify a column of data in the table as the source of VIDs in NebulaGraph.\n # The value of vertex must be consistent with the column name in the above fields or csv.fields.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n vertex: {\n field:_c0\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # The delimiter specified. The default value is comma.\n separator: \",\"\n # If the CSV file has a header, set the header to true.\n # If the CSV file does not have a header, set the header to false. The default value is false.\n header: false\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Whether to repartition data based on the number of partitions of graph spaces in NebulaGraph when generating the SST file.\n repartitionWithNebula: false\n # Set the information about the Tag Team.\n name: team\n type: {\n source: csv\n sink: sst\n path: \"hdfs://*.*.*.*:9000/dataset/vertex_team.csv\"\n fields: [_c1]\n nebula.fields: [name]\n vertex: {\n field:_c0\n separator: \",\"\n header: false\n batch: 256\n partition: 32\n repartitionWithNebula: false\n # If more vertices need to be added, refer to the previous configuration to add them.\n # Processing edges\n edges: [\n # Set the information about the Edge Type follow.\n # The Edge Type name defined in NebulaGraph.\n name: follow\n type: {\n # Specify the data source file format to CSV.\n source: csv\n # Specify how to import the data into NebulaGraph: Client or SST.\n sink: sst\n # Specify the path to the CSV file.\n # If the file is stored in HDFS, use double quotation marks to enclose the file path, starting with hdfs://. For example, \"hdfs://:port/xx/xx.csv\".\n path: \"hdfs://*.*.*.*:9000/dataset/edge_follow.csv\"\n # If the CSV file does not have a header, use [_c0, _c1, _c2, ..., _cn] to represent its header and indicate the columns as the source of the property values.\n # If the CSV file has a header, use the actual column name.\n fields: [_c2]\n # Specify the property name defined in NebulaGraph.\n # The sequence of fields and nebula.fields must correspond to each other.\n nebula.fields: [degree]\n # Specify a column as the source for the source and destination vertices.\n # The value of vertex must be consistent with the column name in the above fields or csv.fields.\n # Currently, NebulaGraph 3.6.0 supports only strings or integers of VID.\n source: {\n field: _c0\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n target: {\n field: _c1\n # Add the specified prefix to the VID. For example, if the VID is `12345`, adding the prefix `tag1` will result in `tag1_12345`. The underscore cannot be modified.\n # prefix:\"tag1\"\n # Performs hashing operations on VIDs of type string.\n # policy:hash\n # The delimiter specified. The default value is comma.\n separator: \",\"\n # (Optional) Specify a column as the source of the rank.\n #ranking: rank\n # If the CSV file has a header, set the header to true.\n # If the CSV file does not have a header, set the header to false. The default value is false.\n header: false\n # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.\n #writeMode: INSERT\n # The number of data written to NebulaGraph in a single batch.\n batch: 256\n # The number of partitions to be created when the data is written to NebulaGraph.\n partition: 32\n # Whether to repartition data based on the number of partitions of graph spaces in NebulaGraph when generating the SST file.\n repartitionWithNebula: false\n # Set the information about the Edge Type serve.\n name: serve\n type: {\n source: csv\n sink: sst\n path: \"hdfs://*.*.*.*:9000/dataset/edge_serve.csv\"\n fields: [_c2,_c3]\n nebula.fields: [start_year, end_year]\n source: {\n field: _c0\n target: {\n field: _c1\n separator: \",\"\n header: false\n batch: 256\n partition: 32\n repartitionWithNebula: false\n # If more edges need to be added, refer to the previous configuration to add them.\nStep 4: Generate the SST file\nRun the following command to generate the SST file from the CSV source file. For a description of the parameters, see Options for import.\n${SPARK_HOME}/bin/spark-submit --master \"local\" --conf spark.sql.shuffle.partition= --class com.vesoft.nebula.exchange.Exchange -c \nFor example:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --conf spark.sql.shuffle.partition=200 --class com.vesoft.nebula.exchange.Exchange /root/nebula-exchange/nebula-exchange/target/nebula-exchange-3.6.0.jar -c /root/nebula-exchange/nebula-exchange/target/classes/sst_application.conf\nAfter the task is complete, you can view the generated SST file in the /sst directory (specified by the nebula.path.remote parameter) on HDFS.\nStep 5: Import the SST file\nConnect to the NebulaGraph database using the client tool and import the SST file as follows:\nRun the following command to select the graph space you created earlier.\n USE basketballplayer;\nRun the following command to download the SST file:\n SUBMIT JOB DOWNLOAD HDFS \"hdfs://:/\";\nFor example:\n SUBMIT JOB DOWNLOAD HDFS \"hdfs://*.*.*.*:9000/sst\";\nRun the following command to import the SST file:\n SUBMIT JOB INGEST;\nStep 6: (Optional) Validate data\nUsers can verify that data has been imported by executing a query in the NebulaGraph client (for example, NebulaGraph Studio). For example:\nLOOKUP ON player YIELD id(vertex);\nUsers can also run the SHOW STATS command to view statistics.\nStep 7: (Conditional) Rebuild indexes in NebulaGraph\nWith the data imported, users can recreate and rebuild indexes in NebulaGraph. For details, see Index overview.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/use-exchange/ex-ug-import-from-sst/","type":"doc"},{"title":"Exchange FAQ","content":"Compilation\nQ: Some packages not in central repository failed to download, error: Could not resolve dependencies for project xxx\nPlease check the mirror part of Maven installation directory libexec/conf/settings.xml:\n\n alimaven\n central\n aliyun maven\n http://maven.aliyun.com/nexus/content/repositories/central/\n\nCheck whether the value of mirrorOf is configured to *. If it is, change it to central or *,!SparkPackagesRepo,!bintray-streamnative-maven.\nReason: There are two dependency packages in Exchange's pom.xml that are not in Maven's central repository. pom.xml configures the repository address for these two dependencies. If the mirrorOf value for the mirror address configured in Maven is *, all dependencies will be downloaded from the Central repository, causing the download to fail.\nQ: Unable to download SNAPSHOT packages when compiling Exchange\nProblem description: The system reports Could not find artifact com.vesoft:client:jar:xxx-SNAPSHOT when compiling.\nCause: There is no local Maven repository for storing or downloading SNAPSHOT packages. The default central repository in Maven only stores official releases, not development versions (SNAPSHOT).\nSolution: Add the following configuration in the profiles scope of Maven's setting.xml file:\n \n \n true\n \n \n \n snapshots\n https://oss.sonatype.org/content/repositories/snapshots/\n \n true\n \n \n \n \nExecution\nQ: Error: java.lang.ClassNotFoundException: com.vesoft.nebula.exchange.Exchange\nTo submit a task in Yarn-Cluster mode, run the following command, especially the two '--conf' commands in the example.\n$SPARK_HOME/bin/spark-submit --class com.vesoft.nebula.exchange.Exchange \\\n--master yarn-cluster \\\n--files application.conf \\\n--conf spark.driver.extraClassPath=./ \\\n--conf spark.executor.extraClassPath=./ \\\nnebula-exchange-3.0.0.jar \\\n-c application.conf\nQ: Error: method name xxx not found\nGenerally, the port configuration is incorrect. Check the port configuration of the Meta service, Graph service, and Storage service.\nQ: Error: NoSuchMethod, MethodNotFound (Exception in thread \"main\" java.lang.NoSuchMethodError, etc)\nMost errors are caused by JAR package conflicts or version conflicts. Check whether the version of the error reporting service is the same as that used in Exchange, especially Spark, Scala, and Hive.\nQ: When Exchange imports Hive data, error: Exception in thread \"main\" org.apache.spark.sql.AnalysisException: Table or view not found\nCheck whether the -h parameter is omitted in the command for submitting the Exchange task and whether the table and database are correct, and run the user-configured exec statement in spark-SQL to verify the correctness of the exec statement.\nQ: Run error: com.facebook.thrift.protocol.TProtocolException: Expected protocol id xxx\nCheck that the NebulaGraph service port is configured correctly.\nFor source, RPM, or DEB installations, configure the port number corresponding to --port in the configuration file for each service.\nFor docker installation, configure the docker mapped port number as follows:Execute docker-compose ps in the nebula-docker-compose directory, for example:\n$ docker-compose ps\n Name Command State Ports\n---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\nnebula-docker-compose_graphd_1 /usr/local/nebula/bin/nebu ... Up (healthy) 0.0.0.0:33205->19669/tcp, 0.0.0.0:33204->19670/tcp, 0.0.0.0:9669->9669/tcp\nnebula-docker-compose_metad0_1 ./bin/nebula-metad --flagf ... Up (healthy) 0.0.0.0:33165->19559/tcp, 0.0.0.0:33162->19560/tcp, 0.0.0.0:33167->9559/tcp, 9560/tcp\nnebula-docker-compose_metad1_1 ./bin/nebula-metad --flagf ... Up (healthy) 0.0.0.0:33166->19559/tcp, 0.0.0.0:33163->19560/tcp, 0.0.0.0:33168->9559/tcp, 9560/tcp\nnebula-docker-compose_metad2_1 ./bin/nebula-metad --flagf ... Up (healthy) 0.0.0.0:33161->19559/tcp, 0.0.0.0:33160->19560/tcp, 0.0.0.0:33164->9559/tcp, 9560/tcp\nnebula-docker-compose_storaged0_1 ./bin/nebula-storaged --fl ... Up (healthy) 0.0.0.0:33180->19779/tcp, 0.0.0.0:33178->19780/tcp, 9777/tcp, 9778/tcp, 0.0.0.0:33183->9779/tcp, 9780/tcp\nnebula-docker-compose_storaged1_1 ./bin/nebula-storaged --fl ... Up (healthy) 0.0.0.0:33175->19779/tcp, 0.0.0.0:33172->19780/tcp, 9777/tcp, 9778/tcp, 0.0.0.0:33177->9779/tcp, 9780/tcp\nnebula-docker-compose_storaged2_1 ./bin/nebula-storaged --fl ... Up (healthy) 0.0.0.0:33184->19779/tcp, 0.0.0.0:33181->19780/tcp, 9777/tcp, 9778/tcp, 0.0.0.0:33185->9779/tcp, 9780/tcp\nCheck the Ports column to find the docker mapped port number, for example:\n- The port number available for Graph service is 9669.\n- The port number for Meta service are 33167, 33168, 33164.\n- The port number for Storage service are 33183, 33177, 33185.\nQ: Error: Exception in thread \"main\" com.facebook.thrift.protocol.TProtocolException: The field 'code' has been assigned the invalid value -4\nCheck whether the version of Exchange is the same as that of NebulaGraph. For more information, see Limitations.\nQ: How to correct the messy code when importing Hive data into NebulaGraph?\nIt may happen if the property value of the data in Hive contains Chinese characters. The solution is to add the following options before the JAR package path in the import command:\n--conf spark.driver.extraJavaOptions=-Dfile.encoding=utf-8\n--conf spark.executor.extraJavaOptions=-Dfile.encoding=utf-8\nNamely:\n/bin/spark-submit --master \"local\" \\\n--conf spark.driver.extraJavaOptions=-Dfile.encoding=utf-8 \\\n--conf spark.executor.extraJavaOptions=-Dfile.encoding=utf-8 \\\n--class com.vesoft.nebula.exchange.Exchange \\\n -c \nIn YARN, use the following command:\n/bin/spark-submit \\\n--class com.vesoft.nebula.exchange.Exchange \\\n--master yarn-cluster \\\n--files \\\n--conf spark.driver.extraClassPath=./ \\\n--conf spark.executor.extraClassPath=./ \\\n--conf spark.driver.extraJavaOptions=-Dfile.encoding=utf-8 \\\n--conf spark.executor.extraJavaOptions=-Dfile.encoding=utf-8 \\\n \\\n-c application.conf\nQ: org.rocksdb.RocksDBException: While open a file for appending: /path/sst/1-xxx.sst: No such file or directory\nSolution:\nCheck if /path exists. If not, or if the path is set incorrectly, create or correct it.\nCheck if Spark's current user on each machine has the operation permission on /path. If not, grant the permission.\nConfiguration\nQ: Which configuration fields will affect import performance?\nbatch: The number of data contained in each nGQL statement sent to the NebulaGraph service.\npartition: The number of partitions to be created when the data is written to NebulaGraph, indicating the number of concurrent data imports.\nnebula.rate: Get a token from the token bucket before sending a request to NebulaGraph.- limit: Represents the size of the token bucket.\n- timeout: Represents the timeout period for obtaining the token.\nThe values of these four parameters can be adjusted appropriately according to the machine performance. If the leader of the Storage service changes during the import process, you can adjust the values of these four parameters to reduce the import speed.\nOthers\nQ: Which versions of NebulaGraph are supported by Exchange?\nSee Limitations.\nQ: What is the relationship between Exchange and Spark Writer?\nExchange is the Spark application developed based on Spark Writer. Both are suitable for bulk migration of cluster data to NebulaGraph in a distributed environment, but later maintenance work will be focused on Exchange. Compared with Spark Writer, Exchange has the following improvements:\nIt supports more abundant data sources, such as MySQL, Neo4j, Hive, HBase, Kafka, Pulsar, etc.\nIt fixed some problems of Spark Writer. For example, when Spark reads data from HDFS, the default source data is String, which may be different from the NebulaGraph's Schema. So Exchange adds automatic data type matching and type conversion. When the data type in the NebulaGraph's Schema is non-String (e.g. double), Exchange converts the source data of String type to the corresponding type.","url":"https://docs.nebula-graph.io/3.6.0/import-export/nebula-exchange/ex-ug-FAQ/","type":"doc"},{"title":"NebulaGraph Spark Connector","content":"NebulaGraph Spark Connector is a Spark connector application for reading and writing NebulaGraph data in Spark standard format. NebulaGraph Spark Connector consists of two parts: Reader and Writer.\nReader\nProvides a Spark SQL interface. This interface can be used to read NebulaGraph data. It reads one vertex or edge type data at a time and assemble the result into a Spark DataFrame.\nWriter\nProvides a Spark SQL interface. This interface can be used to write DataFrames into NebulaGraph in a row-by-row or batch-import way.\nFor more information, see NebulaGraph Spark Connector.\nVersion compatibility\nThe correspondence between the NebulaGraph Spark Connector version, the NebulaGraph core version and the Spark version is as follows.\nSpark Connector version\nNebulaGraph version\nSpark version\nnebula-spark-connector_3.0-3.0-SNAPSHOT.jar\nnightly\n3.x\nnebula-spark-connector_2.2-3.0-SNAPSHOT.jar\nnightly\n2.2.x\nnebula-spark-connector-3.0-SNAPSHOT.jar\nnightly\n2.4.x\nnebula-spark-connector_3.0-3.6.0.jar\n3.x\n3.x\nnebula-spark-connector_2.2-3.6.0.jar\n3.x\n2.2.x\nnebula-spark-connector-3.6.0.jar\n3.x\n2.4.x\nnebula-spark-connector_2.2-3.4.0.jar\n3.x\n2.2.x\nnebula-spark-connector-3.4.0.jar\n3.x\n2.4.x\nnebula-spark-connector_2.2-3.3.0.jar\n3.x\n2.2.x\nnebula-spark-connector-3.3.0.jar\n3.x\n2.4.x\nnebula-spark-connector-3.0.0.jar\n3.x\n2.4.x\nnebula-spark-connector-2.6.1.jar\n2.6.0, 2.6.1\n2.4.x\nnebula-spark-connector-2.6.0.jar\n2.6.0, 2.6.1\n2.4.x\nnebula-spark-connector-2.5.1.jar\n2.5.0, 2.5.1\n2.4.x\nnebula-spark-connector-2.5.0.jar\n2.5.0, 2.5.1\n2.4.x\nnebula-spark-connector-2.1.0.jar\n2.0.0, 2.0.1\n2.4.x\nnebula-spark-connector-2.0.1.jar\n2.0.0, 2.0.1\n2.4.x\nnebula-spark-connector-2.0.0.jar\n2.0.0, 2.0.1\n2.4.x\nUse cases\nNebulaGraph Spark Connector applies to the following scenarios:\nRead data from NebulaGraph for analysis and computation.\nWrite data back to NebulaGraph after analysis and computation.\nMigrate the data of NebulaGraph.\nGraph computing with NebulaGraph Algorithm.\nBenefits\nThe features of NebulaGraph Spark Connector 3.6.0 are as follows:\nSupports multiple connection settings, such as timeout period, number of connection retries, number of execution retries, etc.\nSupports multiple settings for data writing, such as setting the corresponding column as vertex ID, starting vertex ID, destination vertex ID or attributes.\nSupports non-attribute reading and full attribute reading.\nSupports reading NebulaGraph data into VertexRDD and EdgeRDD, and supports non-Long vertex IDs.\nUnifies the extended data source of SparkSQL, and uses DataSourceV2 to extend NebulaGraph data.\nThree write modes, insert, update and delete, are supported. insert mode will insert (overwrite) data, update mode will only update existing data, and delete mode will only delete data.\nRelease note\nRelease\nGet NebulaGraph Spark Connector\nCompile package\nClone repository nebula-spark-connector.\n$ git clone -b release-3.6 https://github.com/vesoft-inc/nebula-spark-connector.git\n2. Enter the nebula-spark-connector directory.\nCompile package. The procedure varies with Spark versions.\n- Spark 2.4\n```bash\n$ mvn clean package -Dmaven.test.skip=true -Dgpg.skip -Dmaven.javadoc.skip=true -pl nebula-spark-connector -am -Pscala-2.11 -Pspark-2.4\n```\n- Spark 2.2\n```bash\n$ mvn clean package -Dmaven.test.skip=true -Dgpg.skip -Dmaven.javadoc.skip=true -pl nebula-spark-connector_2.2 -am -Pscala-2.11 -Pspark-2.2\n```\n- Spark 3.x\n```bash\n$ mvn clean package -Dmaven.test.skip=true -Dgpg.skip -Dmaven.javadoc.skip=true -pl nebula-spark-connector_3.0 -am -Pscala-2.12 -Pspark-3.0\n```\nAfter compilation, a similar file nebula-spark-connector-3.6.0-SHANPSHOT.jar is generated in the directory target of the folder.\nDownload maven remote repository\nDownload\nHow to use\nWhen using NebulaGraph Spark Connector to reading and writing NebulaGraph data, You can refer to the following code.\n# Read vertex and edge data from NebulaGraph.\nspark.read.nebula().loadVerticesToDF()\nspark.read.nebula().loadEdgesToDF()\n# Write dataframe data into NebulaGraph as vertex and edges.\ndataframe.write.nebula().writeVertices()\ndataframe.write.nebula().writeEdges()\nnebula() receives two configuration parameters, including connection configuration and read-write configuration.\nReading data from NebulaGraph\nval config = NebulaConnectionConfig\n .builder()\n .withMetaAddress(\"127.0.0.1:9559\")\n .withConenctionRetry(2)\n .withExecuteRetry(2)\n .withTimeout(6000)\n .build()\nval nebulaReadVertexConfig: ReadNebulaConfig = ReadNebulaConfig\n .builder()\n .withSpace(\"test\")\n .withLabel(\"person\")\n .withNoColumn(false)\n .withReturnCols(List(\"birthday\"))\n .withLimit(10)\n .withPartitionNum(10)\n .build()\nval vertex = spark.read.nebula(config, nebulaReadVertexConfig).loadVerticesToDF()\nval nebulaReadEdgeConfig: ReadNebulaConfig = ReadNebulaConfig\n .builder()\n .withSpace(\"test\")\n .withLabel(\"knows\")\n .withNoColumn(false)\n .withReturnCols(List(\"degree\"))\n .withLimit(10)\n .withPartitionNum(10)\n .build()\nval edge = spark.read.nebula(config, nebulaReadEdgeConfig).loadEdgesToDF()\nNebulaConnectionConfig is the configuration for connecting to the nebula graph, as described below.\nParameter\nRequired\nDescription\nwithMetaAddress\nYes\nSpecifies the IP addresses and ports of all Meta Services. Separate multiple addresses with commas. The format is ip1:port1,ip2:port2,.... Read data is no need to configure withGraphAddress.\nwithConnectionRetry\nNo\nThe number of retries that the NebulaGraph Java Client connected to the NebulaGraph. The default value is 1.\nwithExecuteRetry\nNo\nThe number of retries that the NebulaGraph Java Client executed query statements. The default value is 1.\nwithTimeout\nNo\nThe timeout for the NebulaGraph Java Client request response. The default value is 6000, Unit: ms.\nReadNebulaConfig is the configuration to read NebulaGraph data, as described below.\nParameter\nRequired\nDescription\nwithSpace\nYes\nNebulaGraph space name.\nwithLabel\nYes\nThe Tag or Edge type name within the NebulaGraph space.\nwithNoColumn\nNo\nWhether the property is not read. The default value is false, read property. If the value is true, the property is not read, the withReturnCols configuration is invalid.\nwithReturnCols\nNo\nConfigures the set of properties for vertex or edges to read. the format is List(property1,property2,...), The default value is List(), indicating that all properties are read.\nwithLimit\nNo\nConfigure the number of rows of data read from the server by the NebulaGraph Java Storage Client at a time. The default value is 1000.\nwithPartitionNum\nNo\nConfigures the number of Spark partitions to read the NebulaGraph data. The default value is 100. This value should not exceed the number of slices in the graph space (partition_num).\nWrite data into NebulaGraph\nval config = NebulaConnectionConfig\n .builder()\n .withMetaAddress(\"127.0.0.1:9559\")\n .withGraphAddress(\"127.0.0.1:9669\")\n .withConenctionRetry(2)\n .build()\nval nebulaWriteVertexConfig: WriteNebulaVertexConfig = WriteNebulaVertexConfig \n .builder()\n .withSpace(\"test\")\n .withTag(\"person\")\n .withVidField(\"id\")\n .withVidPolicy(\"hash\")\n .withVidAsProp(true)\n .withUser(\"root\")\n .withPasswd(\"nebula\")\n .withBatch(1000)\n .build() \ndf.write.nebula(config, nebulaWriteVertexConfig).writeVertices()\nval nebulaWriteEdgeConfig: WriteNebulaEdgeConfig = WriteNebulaEdgeConfig \n .builder()\n .withSpace(\"test\")\n .withEdge(\"friend\")\n .withSrcIdField(\"src\")\n .withSrcPolicy(null)\n .withDstIdField(\"dst\")\n .withDstPolicy(null)\n .withRankField(\"degree\")\n .withSrcAsProperty(true)\n .withDstAsProperty(true)\n .withRankAsProperty(true)\n .withUser(\"root\")\n .withPasswd(\"nebula\")\n .withBatch(1000)\n .build()\ndf.write.nebula(config, nebulaWriteEdgeConfig).writeEdges()\nThe default write mode is insert, which can be changed to update or delete via withWriteMode configuration:\nval config = NebulaConnectionConfig\n .builder()\n .withMetaAddress(\"127.0.0.1:9559\")\n .withGraphAddress(\"127.0.0.1:9669\")\n .build()\nval nebulaWriteVertexConfig = WriteNebulaVertexConfig\n .builder()\n .withSpace(\"test\")\n .withTag(\"person\")\n .withVidField(\"id\")\n .withVidAsProp(true)\n .withBatch(1000)\n .withWriteMode(WriteMode.UPDATE)\n .build()\ndf.write.nebula(config, nebulaWriteVertexConfig).writeVertices()\nNebulaConnectionConfig is the configuration for connecting to the nebula graph, as described below.\nParameter\nRequired\nDescription\nwithMetaAddress\nYes\nSpecifies the IP addresses and ports of all Meta Services. Separate multiple addresses with commas. The format is ip1:port1,ip2:port2,....\nwithGraphAddress\nYes\nSpecifies the IP addresses and ports of Graph Services. Separate multiple addresses with commas. The format is ip1:port1,ip2:port2,....\nwithConnectionRetry\nNo\nNumber of retries that the NebulaGraph Java Client connected to the NebulaGraph. The default value is 1.\nWriteNebulaVertexConfig is the configuration of the write vertex, as described below.\nParameter\nRequired\nDescription\nwithSpace\nYes\nNebulaGraph space name.\nwithTag\nYes\nThe Tag name that needs to be associated when a vertex is written.\nwithVidField\nYes\nThe column in the DataFrame as the vertex ID.\nwithVidPolicy\nNo\nWhen writing the vertex ID, NebulaGraph use mapping function, supports HASH only. No mapping is performed by default.\nwithVidAsProp\nNo\nWhether the column in the DataFrame that is the vertex ID is also written as an property. The default value is false. If set to true, make sure the Tag has the same property name as VidField.\nwithUser\nNo\nNebulaGraph user name. If authentication is disabled, you do not need to configure the user name and password.\nwithPasswd\nNo\nThe password for the NebulaGraph user name.\nwithBatch\nYes\nThe number of rows of data written at a time. The default value is 1000.\nwithWriteMode\nNo\nWrite mode. The optional values are insert, update and delete. The default value is insert.\nwithDeleteEdge\nNo\nWhether to delete the related edges synchronously when deleting a vertex. The default value is false. It takes effect when withWriteMode is delete.\nWriteNebulaEdgeConfig is the configuration of the write edge, as described below.\nParameter\nRequired\nDescription\nwithSpace\nYes\nNebulaGraph space name.\nwithEdge\nYes\nThe Edge type name that needs to be associated when a edge is written.\nwithSrcIdField\nYes\nThe column in the DataFrame as the vertex ID.\nwithSrcPolicy\nNo\nWhen writing the starting vertex ID, NebulaGraph use mapping function, supports HASH only. No mapping is performed by default.\nwithDstIdField\nYes\nThe column in the DataFrame that serves as the destination vertex.\nwithDstPolicy\nNo\nWhen writing the destination vertex ID, NebulaGraph use mapping function, supports HASH only. No mapping is performed by default.\nwithRankField\nNo\nThe column in the DataFrame as the rank. Rank is not written by default.\nwithSrcAsProperty\nNo\nWhether the column in the DataFrame that is the starting vertex is also written as an property. The default value is false. If set to true, make sure Edge type has the same property name as SrcIdField.\nwithDstAsProperty\nNo\nWhether column that are destination vertex in the DataFrame are also written as property. The default value is false. If set to true, make sure Edge type has the same property name as DstIdField.\nwithRankAsProperty\nNo\nWhether column in the DataFrame that is the rank is also written as property.The default value is false. If set to true, make sure Edge type has the same property name as RankField.\nwithUser\nNo\nNebulaGraph user name. If authentication is disabled, you do not need to configure the user name and password.\nwithPasswd\nNo\nThe password for the NebulaGraph user name.\nwithBatch\nYes\nThe number of rows of data written at a time. The default value is 1000.\nwithWriteMode\nNo\nWrite mode. The optional values are insert, update and delete. The default value is insert.","url":"https://docs.nebula-graph.io/3.6.0/connector/nebula-spark-connector/","type":"doc"},{"title":"NebulaGraph Flink Connector","content":"NebulaGraph Flink Connector is a connector that helps Flink users quickly access NebulaGraph. NebulaGraph Flink Connector supports reading data from the NebulaGraph database or writing other external data to the NebulaGraph database.\nFor more information, see NebulaGraph Flink Connector.\nUse cases\nNebulaGraph Flink Connector applies to the following scenarios:\nRead data from NebulaGraph for analysis and computation.\nWrite data back to NebulaGraph after analysis and computation.\nMigrate the data of NebulaGraph.\nRelease note\nRelease","url":"https://docs.nebula-graph.io/3.6.0/connector/nebula-flink-connector/","type":"doc"},{"title":"Compaction","content":"This topic gives some information about compaction.\nIn NebulaGraph, Compaction is the most important background process and has an important effect on performance.\nCompaction reads the data that is written on the hard disk, then re-organizes the data structure and the indexes, and then writes back to the hard disk. The read performance can increase by times after compaction. Thus, to get high read performance, trigger compaction (full compaction) manually when writing a large amount of data into Nebula Graph.\nNebulaGraph has two types of compaction: automatic compaction and full compaction.\nAutomatic compaction\nAutomatic compaction is automatically triggered when the system reads data, writes data, or the system restarts. The read performance can increase in a short time. Automatic compaction is enabled by default. But once triggered during peak hours, it can cause unexpected IO occupancy that has an unwanted effect on the performance.\nFull compaction\nFull compaction enables large-scale background operations for a graph space such as merging files, deleting the data expired by TTL. This operation needs to be initiated manually. Use the following statements to enable full compaction:\n USE ;\n SUBMIT JOB COMPACT;\nThe preceding statement returns the job ID. To show the compaction progress, use the following statement:\n SHOW JOB ;\nOperation suggestions\nThese are some operation suggestions to keep Nebula Graph performing well.\nAfter data import is done, run SUBMIT JOB COMPACT.\nRun SUBMIT JOB COMPACT periodically during off-peak hours (e.g. early morning).\nTo control the write traffic limitation for compactions, set the following parameter in the nebula-storaged.conf configuration file.\n# Limit the write rate to 20MB/s.\n--rocksdb_rate_limit=20 (in MB/s)\nFAQ\n\"Where are the logs related to Compaction stored?\"\nBy default, the logs are stored under the LOG file in the /usr/local/nebula/data/storage/nebula/{1}/data/ directory, or similar to LOG.old.1625797988509303. You can find the following content.\n** Compaction Stats [default] **\nLevel Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop\n----------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n L0 2/0 2.46 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.53 0.51 2 0.264 0 0\n Sum 2/0 2.46 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.53 0.51 2 0.264 0 0\n Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0\nIf the number of L0 files is large, the read performance will be greatly affected and compaction can be triggered.\n\"Can I do full compactions for multiple graph spaces at the same time?\"\nYes, you can. But the IO is much larger at this time and the efficiency may be affected.\n\"How much time does it take for full compactions?\"\nWhen rocksdb_rate_limit is set to 20, you can estimate the full compaction time by dividing the hard disk usage by the rocksdb_rate_limit. If you do not set the rocksdb_rate_limit value, the empirical value is around 50 MB/s.\n\"Can I modify --rocksdb_rate_limit dynamically?\"\nNo, you cannot.\n\"Can I stop a full compaction after it starts?\"\nNo, you cannot. When you start a full compaction, you have to wait till it is done. This is the limitation of RocksDB.","url":"https://docs.nebula-graph.io/3.6.0/8.service-tuning/compaction/","type":"doc"},{"title":"Storage load balance","content":"You can use the SUBMIT JOB BALANCE statement to balance the distribution of partitions and Raft leaders, or clear some Storage servers for easy maintenance. For details, see SUBMIT JOB BALANCE.\nBalance leader distribution\nTo balance the raft leaders, run SUBMIT JOB BALANCE LEADER. It will start a job to balance the distribution of all the storage leaders in all graph spaces.\nExample\n SUBMIT JOB BALANCE LEADER;\nRun SHOW HOSTS to check the balance result.\n SHOW HOSTS;","url":"https://docs.nebula-graph.io/3.6.0/8.service-tuning/load-balance/","type":"doc"},{"title":"Graph data modeling suggestions","content":"This topic provides general suggestions for modeling data in NebulaGraph.\nModel for performance\nThere is no perfect method to model in Nebula Graph. Graph modeling depends on the questions that you want to know from the data. Your data drives your graph model. Graph data modeling is intuitive and convenient. Create your data model based on your business model. Test your model and gradually optimize it to fit your business. To get better performance, you can change or re-design your model multiple times.\nDesign and evaluate the most important queries\nUsually, various types of queries are validated in test scenarios to assess the overall capabilities of the system. However, in most production scenarios, there are not many types of frequently used queries. You can optimize the data model based on key queries selected according to the Pareto (80/20) principle.\nFull-graph scanning avoidance\nGraph traversal can be performed after one or more vertices/edges are located through property indexes or VIDs. But for some query patterns, such as subgraph and path query patterns, the source vertex or edge of the traversal cannot be located through property indexes or VIDs. These queries find all the subgraphs that satisfy the query pattern by scanning the whole graph space which will have poor query performance. NebulaGraph does not implement indexing for the graph structures of subgraphs or paths. \nNo predefined bonds between Tags and Edge types\nDefine the bonds between Tags and Edge types in the application, not NebulaGraph. There are no statements that could get the bonds between Tags and Edge types.\nTags/Edge types predefine a set of properties\nWhile creating Tags or Edge types, you need to define a set of properties. Properties are part of the NebulaGraph Schema.\nControl changes in the business model and the data model\nChanges here refer to changes in business models and data models (meta-information), not changes in the data itself.\nSome graph databases are designed to be Schema-free, so their data modeling, including the modeling of the graph topology and properties, can be very flexible. Properties can be re-modeled to graph topology, and vice versa. Such systems are often specifically optimized for graph topology access.\nNebulaGraph 3.6.0 is a strong-Schema (row storage) system, which means that the business model should not change frequently. For example, the property Schema should not change. It is similar to avoiding ALTER TABLE in MySQL.\nOn the contrary, vertices and their edges can be added or deleted at low costs. Thus, the easy-to-change part of the business model should be transformed to vertices or edges, rather than properties.\nFor example, in a business model, people have relatively fixed properties such as age, gender, and name. But their contact, place of visit, trade account, and login device are often changing. The former is suitable for modeling as properties and the latter as vertices or edges.\nSet temporary properties through self-loop edges\nAs a strong Schema system, NebulaGraph does not support List-type properties. And using ALTER TAG costs too much. If you need to add some temporary properties or List-type properties to a vertex, you can first create an edge type with the required properties, and then insert one or more edges that direct to the vertex itself. The figure is as follows.\nTo retrieve temporary properties of vertices, fetch from self-loop edges. For example:\n//Create the edge type and insert the loop property.\n CREATE EDGE IF NOT EXISTS temp(tmp int);\n INSERT EDGE temp(tmp) VALUES \"player100\"->\"player100\"@1:(1);\n INSERT EDGE temp(tmp) VALUES \"player100\"->\"player100\"@2:(2);\n INSERT EDGE temp(tmp) VALUES \"player100\"->\"player100\"@3:(3);\n//After the data is inserted, you can query the loop property by general query statements, for example:\n GO FROM \"player100\" OVER temp YIELD properties(edge).tmp;\n//If you want the results to be returned in the form of a List, you can use a function, for example:\n MATCH (v1:player)-[e:temp]->() return collect(e.tmp);\nOperations on loops are not encapsulated with any syntactic sugars and you can use them just like those on normal edges.\nAbout dangling edges\nA dangling edge is an edge that only connects to a single vertex and only one part of the edge connects to the vertex.\nIn NebulaGraph 3.6.0, dangling edges may appear in the following two cases.\nInsert edges with INSERT EDGE statement before the source vertex or the destination vertex exists.\nDelete vertices with DELETE VERTEX statement and the WITH EDGE option is not used. At this time, the system does not delete the related outgoing and incoming edges of the vertices. There will be dangling edges by default.\nDangling edges may appear in NebulaGraph 3.6.0 as the design allow it to exist. And there is no MERGE statement like openCypher has. The existence of dangling edges depends entirely on the application level. You can use GO and LOOKUP statements to find a dangling edge, but cannot use the MATCH statement to find a dangling edge.\nExamples:\n// Insert an edge that connects two vertices which do not exist in the graph. The source vertex's ID is '11'. The destination vertex's ID is'13'. \n CREATE EDGE IF NOT EXISTS e1 (name string, age int);\n INSERT EDGE e1 (name, age) VALUES \"11\"->\"13\":(\"n1\", 1);\n// Query using the `GO` statement\n GO FROM \"11\" over e1 YIELD properties(edge);\n// Query using the `LOOKUP` statement\n LOOKUP ON e1 YIELD EDGE AS r;\n// Query using the `MATCH` statement\n MATCH ()-[e:e1]->() RETURN e;\nEmpty set (time spent 3153/3573 us)\nBreadth-first traversal over depth-first traversal\nNebulaGraph has lower performance for depth-first traversal based on the Graph topology, and better performance for breadth-first traversal and obtaining properties. For example, if model A contains properties \"name\", \"age\", and \"eye color\", it is recommended to create a tag person and add properties name, age, and eye_color to it. If you create a tag eye_color and an edge type has, and then create an edge to represent the eye color owned by the person, the traversal performance will not be high.\nThe performance of finding an edge by an edge property is close to that of finding a vertex by a vertex property. For some databases, it is recommended to re-model edge properties as those of the intermediate vertices. For example, model the pattern (src)-[edge {P1, P2}]->(dst) as (src)-[edge1]->(i_node {P1, P2})-[edge2]->(dst). With NebulaGraph 3.6.0, you can use (src)-[edge {P1, P2}]->(dst) directly to decrease the depth of the traversal and increase the performance.\nEdge directions\nTo query in the opposite direction of an edge, use the following syntax:\n(dst)<-[edge]-(src) or GO FROM dst REVERSELY.\nIf you do not care about the directions or want to query against both directions, use the following syntax:\n(src)-[edge]-(dst) or GO FROM src BIDIRECT.\nTherefore, there is no need to insert the same edge redundantly in the reversed direction.\nSet tag properties appropriately\nPut a group of properties that are on the same level into the same tag. Different groups represent different concepts.\nUse indexes correctly\nUsing property indexes helps find VIDs through properties, but can lead to great performance reduction. Only use an index when you need to find vertices or edges through their properties.\nDesign VIDs appropriately\nSee VID.\nLong texts\nDo not use long texts to create edge properties. Edge properties are stored twice and long texts lead to greater write amplification. For how edges properties are stored, see Storage architecture. It is recommended to store long texts in HBase or Elasticsearch and store its address in NebulaGraph.\nDynamic graphs (sequence graphs) are not supported\nIn some scenarios, graphs need to have the time information to describe how the structure of the entire graph changes over time.1\nThe Rank field on Edges in NebulaGraph 3.6.0 can be used to store time in int64, but no field on vertices can do this because if you store the time information as property values, it will be covered by new insertion. Thus NebulaGraph does not support sequence graphs.\nFree graph data modeling tools\narrows.app\nhttps://blog.twitter.com/engineering/en_us/topics/insights/2021/temporal-graph-networks ↩","url":"https://docs.nebula-graph.io/3.6.0/8.service-tuning/2.graph-modeling/","type":"doc"},{"title":"System design suggestions","content":"QPS or low-latency first\nNebulaGraph 3.6.0 is good at handling small requests with high concurrency. In such scenarios, the whole graph is huge, containing maybe trillions of vertices or edges, but the subgraphs accessed by each request are not large (containing millions of vertices or edges), and the latency of a single request is low. The concurrent number of such requests, i.e., the QPS, can be huge.\nOn the other hand, in interactive analysis scenarios, the request concurrency is usually not high, but the subgraphs accessed by each request are large, with thousands of millions of vertices or edges. To lower the latency of big requests in such scenarios, you can split big requests into multiple small requests in the application, and concurrently send them to multiple graphd processes. This can decrease the memory used by each graphd process as well. Besides, you can use NebulaGraph Algorithm for such scenarios.\nData transmission and optimization\nRead/write balance. NebulaGraph fits into OLTP scenarios with balanced read/write, i.e., concurrent write and read. It is not suitable for OLAP scenarios that usually need to write once and read many times.\nSelect different write methods. For large batches of data writing, use SST files. For small batches of data writing, use INSERT.\nRun COMPACTION and BALANCE jobs to optimize data format and storage distribution at the right time.\nNebulaGraph 3.6.0 does not support transactions and isolation in the relational database and is closer to NoSQL.\nQuery preheating and data preheating\nPreheat on the application side:\nThe Grapd process does not support pre-compiling queries and generating corresponding query plans, nor can it cache previous query results.\nThe Storagd process does not support preheating data. Only the LSM-Tree and BloomFilter of RocksDB are loaded into memory at startup.\nOnce accessed, vertices and edges are cached respectively in two types of LRU cache of the Storage Service.","url":"https://docs.nebula-graph.io/3.6.0/8.service-tuning/3.system-design/","type":"doc"},{"title":"Execution plan","content":"NebulaGraph 3.6.0 applies rule-based execution plans. Users cannot change execution plans, pre-compile queries (and corresponding plan cache), or accelerate queries by specifying indexes.\nTo view the execution plan and executive summary, see EXPLAIN and PROFILE.","url":"https://docs.nebula-graph.io/3.6.0/8.service-tuning/4.plan/","type":"doc"},{"title":"Processing super vertices","content":"Principle introduction\nIn graph theory, a super vertex, also known as a dense vertex, is a vertex with an extremely high number of adjacent edges. The edges can be outgoing or incoming.\nSuper vertices are very common because of the power-law distribution. For example, popular leaders in social networks (Internet celebrities), top stocks in the stock market, Big Four in the banking system, hubs in transportation networks, websites with high clicking rates on the Internet, and best sellers in E-commerce.\nIn NebulaGraph 3.6.0, a vertex and its properties form a key-value pair, with its VID and other meta information as the key. Its Out-Edge Key-Value and In-Edge Key-Value are stored in the same partition in the form of LSM-trees in hard disks and caches.\nTherefore, directed traversals from this vertex and directed traversals ending at this vertex both involve either a large number of sequential IO scans (ideally, after Compaction or a large number of random IO (frequent writes to the vertex and its ingoing and outgoing edges).\nAs a rule of thumb, a vertex is considered dense when the number of its edges exceeds 10,000. Some special cases require additional consideration.\nIndexes for duplicate properties\nIn a property graph, there is another class of cases similar to super vertices: a property has a very high duplication rate, i.e., many vertices with the same tag but different VIDs have identical property and property values.\nProperty indexes in NebulaGraph 3.6.0 are designed to reuse the functionality of RocksDB in the Storage Service, in which case indexes are modeled as keys with the same prefix. If the lookup of a property fails to hit the cache, it is processed as a random seek and a sequential prefix scan on the hard disk to find the corresponding VID. After that, the graph is usually traversed from this vertex, so that another random read and sequential scan for the corresponding key-value of this vertex will be triggered. The higher the duplication rate, the larger the scan range.\nFor more information about property indexes, see How indexing works in NebulaGraph.\nUsually, special design and processing are required when the number of duplicate property values exceeds 10,000.\nSuggested solutions\nSolutions at the database end\nTruncation: Only return a certain number (a threshold) of edges, and do not return other edges exceeding this threshold.\nCompact: Reorganize the order of data in RocksDB to reduce random reads and increase sequential reads.\nSolutions at the application end\nBreak up some of the super vertices according to their business significance:\nDelete multiple edges and merge them into one.\nFor example, in the transfer scenario (Account_A)-[TRANSFER]->(Account_B), each transfer record is modeled as an edge between account A and account B, then there may be tens of thousands of transfer records between (Account_A) and (Account_B).\nIn such scenarios, merge obsolete transfer details on a daily, weekly, or monthly basis. That is, batch-delete old edges and replace them with a small number of edges representing monthly total and times. And keep the transfer details of the latest month.\nSplit an edge into multiple edges of different types.\nFor example, in the (Airport)<-[DEPART]-(Flight) scenario, the departure of each flight is modeled as an edge between a flight and an airport. Departures from a big airport might be enormous.\nAccording to different airlines, divide the DEPART edge type into finer edge types, such as DEPART_CEAIR, DEPART_CSAIR, etc. Specify the departing airline in queries (graph traversal).\nSplit vertices.\nFor example, in the loan network (person)-[BORROW]->(bank), large bank A will have a very large number of loans and borrowers.\nIn such scenarios, you can split the large vertex A into connected sub-vertices A1, A2, and A3.\n(Person1)-[BORROW]->(BankA1), (Person2)-[BORROW]->(BankA2), (Person2)-[BORROW]->(BankA3);\n(BankA1)-[BELONGS_TO]->(BankA), (BankA2)-[BELONGS_TO]->(BankA), (BankA3)-[BELONGS_TO]->(BankA).\nA1, A2, and A3 can either be three real branches of bank A, such as Beijing branch, Shanghai branch, and Zhejiang branch, or three virtual branches set up according to certain rules, such as A1: 1-1000, A2: 1001-10000 and A3: 10000+ according to the number of loans. In this way, any operation on A is converted into three separate operations on A1, A2, and A3.","url":"https://docs.nebula-graph.io/3.6.0/8.service-tuning/super-node/","type":"doc"},{"title":"Enable AutoFDO for NebulaGraph","content":"The AutoFDO can analyze the performance of an optimized program and use the program's performance information to guide the compiler to re-optimize the program. This document will help you to enable the AutoFDO for NebulaGraph.\nMore information about the AutoFDO, please refer AutoFDO Wiki.\nResource Preparations\nInstall Dependencies\nInstall perf\nsudo apt-get update\nsudo apt-get install -y linux-tools-common \\\nlinux-tools-generic \\\nlinux-tools-`uname -r`\nInstall autofdo tool\nsudo apt-get update\nsudo apt-get install -y autofdo\nOr you can compile the autofdo tool from source.\nNebulaGraph Binary with Debug Version\nFor how to build NebulaGraph from source, please refer to the official document: Install NebulaGraph by compiling the source code.\nIn the configure step, replace CMAKE_BUILD_TYPE=Release with CMAKE_BUILD_TYPE=RelWithDebInfo as below:\n$ cmake -DCMAKE_INSTALL_PREFIX=/usr/local/nebula -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo ..\nPrepare Test Data\nIn our test environment, we use NebulaGraph Bench to prepare the test data and collect the profile data by running the FindShortestPath, Go1Step, Go2Step, Go3Step, InsertPersonScenario 5 scenarios. \nPrepare Profile Data\nCollect Perf Data For AutoFdo Tool\nAfter the test data preparation work done. Collect the perf data for different scenarios.\nGet the pid of storaged, graphd, metad.\n$ nebula.service status all\n[INFO] nebula-metad: Running as 305422, Listening on 9559\n[INFO] nebula-graphd: Running as 305516, Listening on 9669\n[INFO] nebula-storaged: Running as 305707, Listening on 9779\nStart the perf record for nebula-graphd and nebula-storaged. \nperf record -p 305516,305707 -b -e br_inst_retired.near_taken:pp -o ~/FindShortestPath.data\nStart the benchmark test for FindShortestPath scenario.\ncd NebulaGraph-Bench \npython3 run.py stress run -s benchmark -scenario find_path.FindShortestPath -a localhost:9669 --args='-u 100 -i 100000'\nAfter the benchmark finished, end the perf record by Ctrl + c.\nRepeat above steps to collect corresponding profile data for the rest Go1Step, Go2Step, Go3Step and InsertPersonScenario scenarios.\nCreate Gcov File\ncreate_gcov --binary=$NEBULA_HOME/bin/nebula-storaged \\\n--profile=~/FindShortestPath.data \\\n--gcov=~/FindShortestPath-storaged.gcov \\\n-gcov_version=1\ncreate_gcov --binary=$NEBULA_HOME/bin/nebula-graphd \\\n--profile=~/FindShortestPath.data \\\n--gcov=~/FindShortestPath-graphd.gcov \\\n-gcov_version=1\nRepeat for Go1Step, Go2Step, Go3Step and InsertPersonScenario scenarios.\nMerge the Profile Data\nprofile_merger ~/FindShortestPath-graphd.gcov \\\n~/FindShortestPath-storaged.gcov \\\n~/go1step-storaged.gcov \\\n~/go1step-graphd.gcov \\\n~/go2step-storaged.gcov \\\n~/go2step-graphd.gcov \\\n~/go3step-storaged.gcov \\\n~/go3step-master-graphd.gcov \\\n~/InsertPersonScenario-storaged.gcov \\\n~/InsertPersonScenario-graphd.gcov\nYou will get a merged profile which is named fbdata.afdo after that.\nRecompile GraphNebula Binary with the Merged Profile\nRecompile the GraphNebula Binary by passing the profile with compile option -fauto-profile.\ndiff --git a/cmake/nebula/GeneralCompilerConfig.cmake b/cmake/nebula/GeneralCompilerConfig.cmake\n@@ -20,6 +20,8 @@ add_compile_options(-Wshadow)\n add_compile_options(-Wnon-virtual-dtor)\n add_compile_options(-Woverloaded-virtual)\n add_compile_options(-Wignored-qualifiers)\n+add_compile_options(-fauto-profile=~/fbdata.afdo)\nPerformance Test Result\nHardware & Software Environment\nKey\nValue\nCPU Processor#\nSockets\nNUMA\nCPU Type\nIntel(R) Xeon(R) Platinum 8380 CPU @ 2.30GHz\nCores per Processor\n40C80T\nCache\nL1 data: 48KB L1 i: 32KB L2: 1.25MB per physical core L3: shared 60MB per processor\nMemory\nMicron DDR4 3200MT/s 16GB16Micron DDR4 3200MT/s 16GB16\nSSD Disk\nINTEL SSDPE2KE016T8\nSSD R/W Sequential\n3200 MB/s (read) / 2100 MB/s(write)\nNebula Version\nmaster with commit id 51d84a4ed7d2a032a337e3b996c927e3bc5d1415\nKernel\n4.18.0-408.el8.x86_64\nTest Results\nScenario\nAverage Latency(LiB)\nDefault Binary\nOptimized Binary with AutoFDO\nP95 Latency (LiB)\nDefault Binary\nOptimized Binary with AutoFDO\nFindShortestPath\n8072.52\n7260.10\n22102.00\n19108.00\n8034.32\n7218.59\n22060.85\n19006.00\n8079.27\n7257.24\n22147.00\n19053.00\n8087.66\n7221.39\n22143.00\n19050.00\n8044.77\n7239.85\n22181.00\n19055.00\nSTDDEVP\n20.57\n17.34\nSTDDEVP\n41.41\n32.36\nMean\n8063.71\n7239.43\nMean\n22126.77\n19054.40\nSTDDEVP/Mean\n0.26%\n0.24%\nSTDDEVP/Mean\n0.19%\n0.17%\nOpt/Default\n100.00%\n10.22%\nOpt/Default\n100.00%\n13.89%\nGo1Step\n422.53\n418.37\n838.00\n850.00\n432.37\n402.44\n866.00\n815.00\n437.45\n407.98\n874.00\n836.00\n429.16\n408.38\n858.00\n838.00\n446.38\n411.32\n901.00\n837.00\nSTDDEVP\n8.02\n5.20\nSTDDEVP\n20.63\n11.30\nMean\n433.58\n409.70\nMean\n867.40\n835.20\nSTDDEVP/Mean\n1.85%\n1.27%\nSTDDEVP/Mean\n2.38%\n1.35%\nOpt/Default\n100.00%\n5.51%\nOpt/Default\n100.00%\n3.71%\nGo2Step\n2989.93\n2824.29\n10202.00\n9656.95\n2957.22\n2834.55\n10129.00\n9632.40\n2962.74\n2818.62\n10168.40\n9624.70\n2992.39\n2817.27\n10285.10\n9647.50\n2934.85\n2834.91\n10025.00\n9699.65\nSTDDEVP\n21.53\n7.57\nSTDDEVP\n85.62\n26.25\nMean\n2967.43\n2825.93\nMean\n10161.90\n9652.24\nSTDDEVP/Mean\n0.73%\n0.27%\nSTDDEVP/Mean\n0.84%\n0.27%\nOpt/Default\n100.00%\n4.77%\nOpt/Default\n100.00%\n5.02%\nGo3Step\n93551.97\n89406.96\n371359.55\n345433.50\n92418.43\n89977.25\n368868.00\n352375.20\n92587.67\n90339.25\n365390.15\n356198.55\n93371.64\n92458.95\n373578.15\n365177.75\n94046.05\n89943.44\n373392.25\n352576.00\nSTDDEVP\n609.07\n1059.54\nSTDDEVP\n3077.38\n6437.52\nMean\n93195.15\n90425.17\nMean\n370517.62\n354352.20\nSTDDEVP/Mean\n0.65%\n1.17%\nSTDDEVP/Mean\n0.83%\n1.82%\nOpt/Default\n100.00%\n2.97%\nOpt/Default\n100.00%\n4.36%\nInsertPerson\n2022.86\n1937.36\n2689.00\n2633.45\n1966.05\n1935.41\n2620.45\n2555.00\n1985.25\n1953.58\n2546.00\n2593.00\n2026.73\n1887.28\n2564.00\n2394.00\n2007.55\n1964.41\n2676.00\n2581.00\nSTDDEVP\n23.02\n26.42\nSTDDEVP\n57.45\n82.62\nMean\n2001.69\n1935.61\nMean\n2619.09\n2551.29\nSTDDEVP/Mean\n1.15%\n1.37%\nSTDDEVP/Mean\n2.19%\n3.24%\nOpt/Default\n100.00%\n3.30%\nOpt/Default\n100.00%\n2.59%","url":"https://docs.nebula-graph.io/3.6.0/8.service-tuning/enable_autofdo_for_nebulagraph/","type":"doc"},{"title":"Best practices","content":"NebulaGraph is used in a variety of industries. This topic presents a few best practices for using NebulaGraph. For more best practices, see Blog.\nScenarios\nUse cases\nUser review\nPerformance\nKernel\nWhat is a graph database and what are its use cases - Definition, examples & trends\nNebulaGraph Source Code Explained: Variable-Length Pattern Matching\nAdding a Test Case for NebulaGraph\nBDD-Based Integration Testing Framework for NebulaGraph: Part Ⅰ\nBDD-Based Integration Testing Framework for NebulaGraph: Part II\nUnderstanding Subgraph in NebulaGraph\nFull-Text Indexing in NebulaGraph\nEcosystem tool\nValidating Import Performance of NebulaGraph Importer\nEcosystem Tools: NebulaGraph Dashboard for Monitoring\nVisualizing Graph Data with NebulaGraph Explorer","url":"https://docs.nebula-graph.io/3.6.0/8.service-tuning/practice/","type":"doc"},{"title":"Clients overview","content":"NebulaGraph supports multiple types of clients for users to connect to and manage the NebulaGraph database.\nNebulaGraph Console: the native CLI client\nNebulaGraph CPP: the NebulaGraph client for C++\nNebulaGraph Java: the NebulaGraph client for Java\nNebulaGraph Python: the NebulaGraph client for Python\nNebulaGraph Go: the NebulaGraph client for Golang","url":"https://docs.nebula-graph.io/3.6.0/14.client/1.nebula-client/","type":"doc"},{"title":"NebulaGraph Console","content":"NebulaGraph Console is a native CLI client for NebulaGraph. It can be used to connect a NebulaGraph cluster and execute queries. It also supports special commands to manage parameters, export query results, import test datasets, etc.\nCompatibility with NebulaGraph\nSee github.\nObtain NebulaGraph Console\nYou can obtain NebulaGraph Console in the following ways:\nDownload the binary file from the GitHub releases page.\nCompile the source code to obtain the binary file. For more information, see Install from source code.\nNebulaGraph Console functions\nConnect to NebulaGraph\nTo connect to NebulaGraph with the nebula-console file, use the following syntax:\n -addr -port -u -p \npath_of_console indicates the storage path of the NebulaGraph Console binary file.\nWhen two-way authentication is required after SSL encryption is enabled, you need to specify SSL-related parameters when connecting.\nFor example:\nDirect link to NebulaGraph\n./nebula-console -addr 192.168.8.100 -port 9669 -u root -p nebula\nEnable SSL encryption and require two-way authentication\n./nebula-console -addr 192.168.8.100 -port 9669 -u root -p nebula -enable_ssl -ssl_root_ca_path /home/xxx/cert/root.crt -ssl_cert_path /home/xxx/cert/client.crt -ssl_private_key_path /home/xxx/cert/client.key\nParameter descriptions are as follows:\nParameter\nDescription\n-h/-help\nShows the help menu.\n-addr/-address\nSets the IP or hostname of the Graph service. The default address is 127.0.0.1. \n-P/-port\nSets the port number of the graphd service. The default port number is 9669.\n-u/-user\nSets the username of your NebulaGraph account. Before enabling authentication, you can use any existing username. The default username is root.\n-p/-password\nSets the password of your NebulaGraph account. Before enabling authentication, you can use any characters as the password.\n-t/-timeout\nSets an integer-type timeout threshold of the connection. The unit is millisecond. The default value is 120.\n-e/-eval\nSets a string-type nGQL statement. The nGQL statement is executed once the connection succeeds. The connection stops after the result is returned.\n-f/-file\nSets the path of an nGQL file. The nGQL statements in the file are executed once the connection succeeds. The result will be returned and the connection stops then.\n-enable_ssl\nEnables SSL encryption when connecting to NebulaGraph.\n-ssl_root_ca_path\nSets the storage path of the certification authority file.\n-ssl_cert_path\nSets the storage path of the certificate file.\n-ssl_private_key_path\nSets the storage path of the private key file.\n-ssl_insecure_skip_verify\nSpecifies whether the client skips verifying the server's certificate chain and hostname. The default is false. If set to true, any certificate chain and hostname provided by the server is accepted.\nFor information on more parameters, see the project repository.\nManage parameters\nYou can save parameters for parameterized queries.\nThe command to save a parameter is as follows:\n :param => ;\nThe example is as follows:\n :param p1 => \"Tim Duncan\";\n MATCH (v:player{name:$p1})-[:follow]->(n) RETURN v,n;\n :param p2 => {\"a\":3,\"b\":false,\"c\":\"Tim Duncan\"};\n RETURN $p2.b AS b;\nThe command to view the saved parameters is as follows:\n :params;\nThe command to view the specified parameters is as follows:\n :params ;\nThe command to delete a specified parameter is as follows:\n :param =>;\nExport query results\nExport query results, which can be saved as a CSV file, DOT file, and a format of Profile or Explain.\nThe command to export a csv file is as follows:\n :CSV ;\nThe command to export a DOT file is as follows:\n :dot \nThe example is as follows:\n :dot a.dot\n PROFILE FORMAT=\"dot\" GO FROM \"player100\" OVER follow;\nThe command to export a PROFILE or EXPLAIN format is as follows: \n :profile ;\nor\n :explain ;\nThe example is as follows:\n :profile profile.log\n PROFILE GO FROM \"player102\" OVER serve YIELD dst(edge);\n :profile profile.dot\n PROFILE FORMAT=\"dot\" GO FROM \"player102\" OVER serve YIELD dst(edge);\n :explain explain.log\n EXPLAIN GO FROM \"player102\" OVER serve YIELD dst(edge);\nImport a testing dataset\nThe testing dataset is named basketballplayer. To view details about the schema and data, use the corresponding SHOW command.\nThe command to import a testing dataset is as follows:\n :play basketballplayer\nRun a command multiple times\nTo run a command multiple times, use the following command:\n :repeat N\nThe example is as follows:\n :repeat 3\n GO FROM \"player100\" OVER follow YIELD dst(edge);\nGot 2 rows (time spent 2602/3214 us)\nFri, 20 Aug 2021 06:36:05 UTC\nGot 2 rows (time spent 583/849 us)\nFri, 20 Aug 2021 06:36:05 UTC\nGot 2 rows (time spent 496/671 us)\nFri, 20 Aug 2021 06:36:05 UTC\nExecuted 3 times, (total time spent 3681/4734 us), (average time spent 1227/1578 us)\nSleep\nThis command will make NebulaGraph Console sleep for N seconds. The schema is altered in an async way and takes effect in the next heartbeat cycle. Therefore, this command is usually used when altering schema. The command is as follows:\n :sleep N\nDisconnect NebulaGraph Console from NebulaGraph\nYou can use :EXIT or :QUIT to disconnect from NebulaGraph. For convenience, NebulaGraph Console supports using these commands in lower case without the colon (\":\"), such as quit.\nThe example is as follows:\n :QUIT\nBye root!","url":"https://docs.nebula-graph.io/3.6.0/nebula-console/","type":"doc"},{"title":"NebulaGraph CPP","content":"NebulaGraph CPP is a C++ client for connecting to and managing the NebulaGraph database.\nLimitations\nYou have installed C++ and GCC 4.8 or later versions.\nCompatibility with NebulaGraph\nSee github.\nInstall NebulaGraph CPP\nThis document describes how to install NebulaGraph CPP with the source code.\nPrerequisites\nYou have prepared the correct resources.\nYou have installed C++ and GCC version is: {10.1.0 | 9.3.0 | 9.2.0 | 9.1.0 | 8.3.0 | 7.5.0 | 7.1.0}. For details, see the gcc_preset_versions parameter.\nSteps\nClone the NebulaGraph CPP source code to the host.\n(Recommended) To install a specific version of NebulaGraph CPP, use the Git option --branch to specify the branch. For example, to install v3.4.0, run the following command:\n$ git clone --branch release-3.4 https://github.com/vesoft-inc/nebula-cpp.git\nTo install the daily development version, run the following command to download the source code from the master branch:\n$ git clone https://github.com/vesoft-inc/nebula-cpp.git\nChange the working directory to nebula-cpp.\n$ cd nebula-cpp\nCreate a directory named build and change the working directory to it.\n$ mkdir build && cd build\nGenerate the makefile file with CMake.\n$ cmake -DCMAKE_BUILD_TYPE=Release ..\nCompile NebulaGraph CPP.\nTo speed up the compiling, use the -j option to set a concurrent number N. It should be \\(\\min(\\text{CPU core number},\\frac{\\text{the memory size(GB)}}{2})\\).\n$ make -j{N}\nInstall NebulaGraph CPP.\n$ sudo make install\nUpdate the dynamic link library.\n$ sudo ldconfig\nUse NebulaGraph CPP\nCompile the CPP file to an executable file, then you can use it. The following steps take using SessionExample.cpp for example.\nUse the example code to create the SessionExample.cpp file.\nRun the following command to compile the file.\n$ LIBRARY_PATH=:$LIBRARY_PATH g++ -std=c++11 SessionExample.cpp -I -lnebula_graph_client -o session_example\nlibrary_folder_path: The storage path of the NebulaGraph dynamic libraries. The default path is /usr/local/nebula/lib64.\ninclude_folder_path: The storage of the NebulaGraph header files. The default path is /usr/local/nebula/include.\nFor example:\n$ LIBRARY_PATH=/usr/local/nebula/lib64:$LIBRARY_PATH g++ -std=c++11 SessionExample.cpp -I/usr/local/nebula/include -lnebula_graph_client -o session_example\nCore of the example code\nNebula CPP clients provide both Session Pool and Connection Pool methods to connect to NebulaGraph. Using the Connection Pool method requires users to manage session instances by themselves.\nSession Pool\nFor more details about all the code, see SessionPoolExample.\nConnection Pool\nFor more details about all the code, see SessionExample.","url":"https://docs.nebula-graph.io/3.6.0/14.client/3.nebula-cpp-client/","type":"doc"},{"title":"NebulaGraph Java","content":"NebulaGraph Java is a Java client for connecting to and managing the NebulaGraph database.\nPrerequisites\nYou have installed Java 8.0 or later versions.\nCompatibility with NebulaGraph\nSee github.\nDownload NebulaGraph Java\n(Recommended) To install a specific version of NebulaGraph Java, use the Git option --branch to specify the branch. For example, to install v3.6.1, run the following command:\n$ git clone --branch release-3.6 https://github.com/vesoft-inc/nebula-java.git\nTo install the daily development version, run the following command to download the source code from the master branch:\n$ git clone https://github.com/vesoft-inc/nebula-java.git\nUse NebulaGraph Java\nWhen importing a Maven project with tools such as IDEA, set the following dependency in pom.xml.\n\n com.vesoft\n client\n 3.0.0-SNAPSHOT\n\nIf you cannot download the dependency for the daily development version, set the following content in pom.xml. Released versions have no such issue.\n \n \n snapshots \n https://oss.sonatype.org/content/repositories/snapshots/ \n \n\nIf there is no Maven to manage the project, manually download the JAR file to install NebulaGraph Java.\nCore of the example code\nThe NebulaGraph Java client provides both Connection Pool and Session Pool modes, using Connection Pool requires the user to manage session instances.\nSession Pool\nFor all the code, see GraphSessionPoolExample.\nConnection Pool\nFor all the code, see GraphClientExample.","url":"https://docs.nebula-graph.io/3.6.0/14.client/4.nebula-java-client/","type":"doc"},{"title":"NebulaGraph Python","content":"NebulaGraph Python is a Python client for connecting to and managing the NebulaGraph database.\nPrerequisites\nYou have installed Python 3.6 or later versions.\nCompatibility with NebulaGraph\nSee github.\nInstall NebulaGraph Python\nInstall NebulaGraph Python with pip\n$ pip install nebula3-python==\nInstall NebulaGraph Python from the source code\nClone the NebulaGraph Python source code to the host.\n(Recommended) To install a specific version of NebulaGraph Python, use the Git option --branch to specify the branch. For example, to install v3.4.0, run the following command:\n$ git clone --branch release-3.4 https://github.com/vesoft-inc/nebula-python.git\nTo install the daily development version, run the following command to download the source code from the master branch:\n$ git clone https://github.com/vesoft-inc/nebula-python.git\nChange the working directory to nebula-python.\n$ cd nebula-python\nRun the following command to install NebulaGraph Python.\n$ pip install .\nCore of the example code\nNebulaGraph Python clients provides Connection Pool and Session Pool methods to connect to NebulaGraph. Using the Connection Pool method requires users to manage sessions by themselves.\nSession Pool\nFor details about all the code, see SessinPoolExample.py.\nFor limitations of using the Session Pool method, see Example of using session pool.\nConnection Pool\nFor details about all the code, see Example.","url":"https://docs.nebula-graph.io/3.6.0/14.client/5.nebula-python-client/","type":"doc"},{"title":"NebulaGraph Go","content":"NebulaGraph Go is a Golang client for connecting to and managing the NebulaGraph database.\nPrerequisites\nYou have installed Golang 1.13 or later versions.\nCompatibility with NebulaGraph\nSee github.\nDownload NebulaGraph Go\n(Recommended) To install a specific version of NebulaGraph Go, use the Git option --branch to specify the branch. For example, to install v3.6.1, run the following command:\n$ git clone --branch release-3.6 https://github.com/vesoft-inc/nebula-go.git\nTo install the daily development version, run the following command to download the source code from the master branch:\n$ git clone https://github.com/vesoft-inc/nebula-go.git\nInstall or update\nRun the following command to install or update NebulaGraph Go:\n$ go get -u -v github.com/vesoft-inc/nebula-go/v3@v3.6.1\nCore of the example code\nThe NebulaGraph GO client provides both Connection Pool and Session Pool, using Connection Pool requires the user to manage the session instances.\nSession Pool \nFor details about all the code, see session_pool_example.go.\nFor limitations of using Session Pool, see Usage example.\nConnection Pool\nFor all the code, see graph_client_basic_example and graph_client_goroutines_example.","url":"https://docs.nebula-graph.io/3.6.0/14.client/6.nebula-go-client/","type":"doc"},{"title":"What is NebulaGraph Studio","content":"NebulaGraph Studio (Studio in short) is a browser-based visualization tool to manage NebulaGraph. It provides you with a graphical user interface to manipulate graph schemas, import data, and run nGQL statements to retrieve data. With Studio, you can quickly become a graph exploration expert from scratch. You can view the latest source code in the NebulaGraph GitHub repository, see nebula-studio for details.\nDeployment\nIn addition to deploying Studio with RPM-based, DEB-based, or Tar-based packages, or with Docker, you can also deploy Studio with Helm in the Kubernetes cluster. For more information, see Deploy Studio.\nThe functions of the above four deployment methods are the same and may be restricted when using Studio. For more information, see Limitations.\nFeatures\nStudio can easily manage NebulaGraph data, with the following functions:\nOn the Schema page, you can use the graphical user interface to create the space, Tag, Edge Type, Index, and view the statistics on the graph. It helps you quickly get started with NebulaGraph.\nOn the Import page, you can operate batch import of vertex and edge data with clicks, and view a real-time import log.\nOn the Console page, you can run nGQL statements and read the results in a human-friendly way.\nScenarios\nYou can use Studio in one of these scenarios:\nYou have a dataset, and you want to explore and analyze data in a visualized way. You can use Docker Compose to deploy NebulaGraph and then use Studio to explore or analyze data in a visualized way. \nYou are a beginner of nGQL (NebulaGraph Query Language) and you prefer to use a GUI rather than a command-line interface (CLI) to learn the language. \nAuthentication\nAuthentication is not enabled in NebulaGraph by default. Users can log into Studio with the root account and any password.\nWhen NebulaGraph enables authentication, users can only sign into Studio with the specified account. For more information, see Authentication.\nVersion compatibility\nNebulaGraph version\nStudio version\n3.6.0\n3.8.0, 3.7.0\n3.5.0\n3.7.0\n3.4.0 ~ 3.4.1\n3.7.0、3.6.0、3.5.1、3.5.0\n3.3.0\n3.5.1、3.5.0\n3.0.0 ～ 3.2.0\n3.4.1、3.4.0\n3.1.0\n3.3.2\n3.0.0\n3.2.x\n2.6.x\n3.1.x\n2.6.x\n3.1.x\n2.0 & 2.0.1\n2.x\n1.x\n1.x\nCheck updates\nStudio is in development. Users can view the latest releases features through Changelog.\nTo view the Changelog, on the upper-right corner of the page, click the version and then New version.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/about-studio/st-ug-what-is-graph-studio/","type":"doc"},{"title":"Limitations","content":"This topic introduces the limitations of Studio.\nArchitecture\nFor now, Studio v3.x supports x86_64 architecture only.\nUpload data\nOnly CSV files without headers can be uploaded, but no limitations are applied to the size and store period for a single file. The maximum data volume depends on the storage capacity of your machine.\nData backup\nFor now, only supports exporting query results in CSV format on Console, and other data backup methods are not supported.\nnGQL statements\nOn the Console page of Docker-based and RPM-based Studio v3.x, all the nGQL syntaxes except these are supported:\nUSE : You cannot run such a statement on the Console page to choose a graph space. As an alternative, you can click a graph space name in the drop-down list of Current Graph Space.\nYou cannot use line breaks (\\). As an alternative, you can use the Enter key to split a line.\nBrowser\nWe recommend that you use the latest version of Chrome to get access to Studio. Otherwise, some features may not work properly.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/about-studio/st-ug-limitations/","type":"doc"},{"title":"Deploy Studio","content":"This topic describes how to deploy Studio locally by RPM, DEB, tar package and Docker.\nRPM-based Studio\nPrerequisites\nBefore you deploy RPM-based Studio, you must confirm that:\nThe NebulaGraph services are deployed and started. For more information, see NebulaGraph Database Manual.\nThe Linux distribution is CentOS, install lsof.\nBefore the installation starts, the following ports are not occupied.\nPort\nDescription\n7001\nWeb service provided by Studio.\nInstall\nSelect and download the RPM package according to your needs. It is recommended to select the latest version. Common links are as follows:\nInstallation package\nChecksum\nNebulaGraph version\nnebula-graph-studio-3.8.0.x86_64.rpm\nnebula-graph-studio-3.8.0.x86_64.rpm.sha256\n3.6.0\nUse sudo rpm -i to install RPM package.\nFor example, install Studio 3.8.0, use the following command. The default installation path is /usr/local/nebula-graph-studio.\n $ sudo rpm -i nebula-graph-studio-3.8.0.x86_64.rpm\nYou can also install it to the specified path using the following command:\n $ sudo rpm -i nebula-graph-studio-3.8.0.x86_64.rpm --prefix= \nWhen the screen returns the following message, it means that the PRM-based Studio has been successfully started.\nStart installing NebulaGraph Studio now...\nNebulaGraph Studio has been installed.\nNebulaGraph Studio started automatically.\nWhen Studio is started, use http://:7001 to get access to Studio.\nIf you can see the Config Server page on the browser, Studio is started successfully.\nUninstall\nYou can uninstall Studio using the following command:\n$ sudo rpm -e nebula-graph-studio-3.8.0.x86_64\nIf these lines are returned, PRM-based Studio has been uninstalled.\nNebulaGraph Studio removed, bye~\nException handling\nIf the automatic start fails during the installation process or you want to manually start or stop the service, use the following command:\nStart the service manually\n$ bash /usr/local/nebula-graph-studio/scripts/rpm/start.sh\nStop the service manually\n$ bash /usr/local/nebula-graph-studio/scripts/rpm/stop.sh\nIf you encounter an error bind EADDRINUSE 0.0.0.0:7001 when starting the service, you can use the following command to check port 7001 usage.\n$ lsof -i:7001\nIf the port is occupied and the process on that port cannot be terminated, you can modify the startup port within the studio configuration and restart the service.\n//Modify the studio service configuration. The default path to the configuration file is `/usr/local/nebula-graph-studio`.\n$ vi etc/studio-api.yam\n//Modify this port number and change it to any \nPort: 7001\n//Restart service\n$ systemctl restart nebula-graph-studio.service\nDEB-based Studio\nPrerequisites\nBefore you deploy DEB-based Studio, you must do a check of these:\nThe NebulaGraph services are deployed and started. For more information, see NebulaGraph Database Manual.\nThe Linux distribution is Ubuntu.\nBefore the installation starts, the following ports are not occupied.\nPort\nDescription\n7001\nWeb service provided by Studio\nThe path /usr/lib/systemd/system exists in the system. If not, create it manually.\nInstall\nSelect and download the DEB package according to your needs. It is recommended to select the latest version. Common links are as follows:\nInstallation package\nChecksum\nNebulaGraph version\nnebula-graph-studio-3.8.0.x86_64.deb\nnebula-graph-studio-3.8.0.x86_64.deb.sha256\n3.6.0\nUse sudo dpkg -i to install DEB package.\nFor example, install Studio 3.8.0, use the following command:\n$ sudo dpkg -i nebula-graph-studio-3.8.0.x86_64.deb\nWhen Studio is started, use http://:7001 to get access to Studio.\nIf you can see the Config Server page on the browser, Studio is started successfully.\nUninstall\nYou can uninstall Studio using the following command:\n$ sudo dpkg -r nebula-graph-studio\ntar-based Studio\nPrerequisites\nBefore you deploy tar-based Studio, you must do a check of these:\nThe NebulaGraph services are deployed and started. For more information, see NebulaGraph Database Manual.\nBefore the installation starts, the following ports are not occupied.\nPort\nDescription\n7001\nWeb service provided by Studio\nInstall and deploy\nSelect and download the tar package according to your needs. It is recommended to select the latest version. Common links are as follows:\nInstallation package\nStudio version\nnebula-graph-studio-3.8.0.x86_64.tar.gz\n3.8.0\nUse tar -xvf to decompress the tar package.\n$ tar -xvf nebula-graph-studio-3.8.0.x86_64.tar.gz\nDeploy and start nebula-graph-studio.\n$ cd nebula-graph-studio\n$ ./server\nWhen Studio is started, use http://:7001 to get access to Studio.\nIf you can see the Config Server page on the browser, Studio is started successfully.\nStop Service\nYou can use kill pid to stop the service:\n$ kill $(lsof -t -i :7001) #stop nebula-graph-studio\nDocker-based Studio\nPrerequisites\nBefore you deploy Docker-based Studio, you must do a check of these:\nThe NebulaGraph services are deployed and started. For more information, see NebulaGraph Database Manual.\nOn the machine where Studio will run, Docker Compose is installed and started. For more information, see Docker Compose Documentation.\nBefore the installation starts, the following ports are not occupied.\nPort\nDescription\n7001\nWeb service provided by Studio\nProcedure\nTo deploy and start Docker-based Studio, run the following commands. Here we use NebulaGraph v3.6.0 for demonstration:\nDownload the configuration files for the deployment.\nInstallation package\nNebulaGraph version\nnebula-graph-studio-3.8.0.tar.gz\n3.6.0\nCreate the nebula-graph-studio-3.8.0 directory and decompress the installation package to the directory.\n$ mkdir nebula-graph-studio-3.8.0 -zxvf nebula-graph-studio-3.8.0.gz -C nebula-graph-studio-3.8.0\nChange to the nebula-graph-studio-3.8.0 directory.\n $ cd nebula-graph-studio-3.8.0\nPull the Docker image of Studio.\n$ docker-compose pull\nBuild and start Docker-based Studio. In this command, -d is to run the containers in the background.\n$ docker-compose up -d\nIf these lines are returned, Docker-based Studio v3.x is deployed and started.\nCreating docker_web_1 ... done\nWhen Docker-based Studio is started, use http://:7001 to get access to Studio.\nIf you can see the Config Server page on the browser, Docker-based Studio is started successfully.\nHelm-based Studio\nThis section describes how to deploy Studio with Helm.\nPrerequisites\nBefore installing Studio, you need to install the following software and ensure the correct version of the software:\nSoftware\nRequirement\nKubernetes\n>= 1.14\nHelm\n>= 3.2.0\nInstall\nUse Git to clone the source code of Studio to the host.\n$ git clone https://github.com/vesoft-inc/nebula-studio.git\nMake the nebula-studio directory the current working directory.\nbash\n $ cd nebula-studio\nAssume using release name:my-studio, installed Studio in Helm Chart.\n$ helm upgrade --install my-studio --set service.type=NodePort --set service.port=30070deployment/helm\nThe configuration parameters of the Helm Chart are described below.\nParameter\nDefault value\nDescription\nreplicaCount\nThe number of replicas for Deployment.\nimage.nebulaStudio.name\nvesoft/nebula-graph-studio\nThe image name of nebula-graph-studio.\nimage.nebulaStudio.version\nv3.8.0\nThe image version of nebula-graph-studio.\nservice.type\nClusterIP\nThe service type, which should be one of NodePort, ClusterIP, and LoadBalancer.\nservice.port\n7001\nThe expose port for nebula-graph-studio's web.\nservice.nodePort\n32701\nThe proxy port for accessing nebula-studio outside kubernetes cluster.\nresources.nebulaStudio\n{}\nThe resource limits/requests for nebula-studio.\npersistent.storageClassName\n\"\"\nThe name of storageClass. The default value will be used if not specified.\npersistent.size\n5Gi\nThe persistent volume size.\nWhen Studio is started, use http://:30070/ to get access to Studio.\nIf you can see the Config Server page on the browser, Studio is started successfully.\nUninstall\n $ helm uninstall my-studio\nNext to do\nOn the Config Server page, connect Docker-based Studio to NebulaGraph. For more information, see Connect to NebulaGraph.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/deploy-connect/st-ug-deploy/","type":"doc"},{"title":"Connect to NebulaGraph","content":"After successfully launching Studio, you need to configure to connect to NebulaGraph. This topic describes how Studio connects to the NebulaGraph database.\nPrerequisites\nBefore connecting to the NebulaGraph database, you need to confirm the following information:\nThe NebulaGraph services and Studio are started. For more information, see Deploy Studio.\nYou have the local IP address and the port used by the Graph service of NebulaGraph. The default port is 9669. \nYou have a NebulaGraph account and its password.\nProcedure\nTo connect Studio to NebulaGraph, follow these steps:\nType http://:7001 in the address bar of your browser.\nThe following login page shows that Studio starts successfully.\nOn the Config Server page of Studio, configure these fields:\nGraphd IP address: Enter the IP address of the Graph service of NebulaGraph. For example, 192.168.10.100.\nPort: The port of the Graph service. The default port is 9669.\nUsername and Password: Fill in the log in account according to the authentication settings of NebulaGraph.\nIf authentication is not enabled, you can use root and any password as the username and its password.\nIf authentication is enabled and no account information has been created, you can only log in as GOD role and use root and nebula as the username and its password.\nIf authentication is enabled and different users are created and assigned roles, users in different roles log in with their accounts and passwords.\nAfter the configuration, click the Connect button.\nA welcome page is displayed on the first login, showing the relevant functions according to the usage process, and the test datasets can be automatically downloaded and imported.\nTo visit the welcome page, click .\nNext to do\nWhen Studio is successfully connected to NebulaGraph, you can do these operations:\nCreate a schema on the Console page or on the Schema page.\nBatch import data on the Import page.\nExecute nGQL statements on the Console page.\nDesign the schema visually on the Schema drafting page.\nLog out\nIf you want to reconnect to NebulaGraph, you can log out and reconfigure the database.\nClick the user profile picture in the upper right corner, and choose Log out.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/deploy-connect/st-ug-connect/","type":"doc"},{"title":"Design a schema","content":"To manipulate graph data in NebulaGraph with Studio, you must have a graph schema. This article introduces how to design a graph schema for NebulaGraph.\nA graph schema for NebulaGraph must have these essential elements:\nTags (namely vertex types) and their properties.\nEdge types and their properties.\nIn this article, you can install the sample data set basketballplayer and use it to explore a pre-designed schema.\nThis table gives all the essential elements of the schema.\nElement\nName\nProperty name (Data type)\nDescription\nTag\nplayer\n- name (string) - age (int)\nRepresents the player.\nTag\nteam\n- name (string)\nRepresents the team.\nEdge type\nserve\n- start_year (int) - end_year (int)\nRepresent the players behavior.This behavior connects the player to the team, and the direction is from player to team.\nEdge type\nfollow\n- degree (int)\nRepresent the players behavior.This behavior connects the player to the player, and the direction is from a player to a player.\nThis figure shows the relationship (serve/follow) between a player and a team.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/quick-start/st-ug-plan-schema/","type":"doc"},{"title":"Create a schema","content":"To batch import data into NebulaGraph, you must have a graph schema. You can create a schema on the Console page or on the Schema page of Studio.\nPrerequisites\nTo create a graph schema on Studio, you must do a check of these:\nStudio is connected to NebulaGraph.\nYour account has the privilege of GOD, ADMIN, or DBA.\nThe schema is designed.\nA graph space is created.\nCreate a schema with Schema\nCreate tags. For more information, see Operate tags.\nCreate edge types. For more information, see Operate edge types.\nCreate a schema with Console\nIn the toolbar, click the Console tab.\nIn the Current Graph Space field, choose a graph space name. In this example, basketballplayer is used.\nIn the input box, enter these statements one by one and click the button Run.\n// To create a tag named \"player\", with two property\n CREATE TAG player(name string, age int);\n// To create a tag named \"team\", with one property\n CREATE TAG team(name string);\n// To create an edge type named \"follow\", with one properties\n CREATE EDGE follow(degree int);\n// To create an edge type named \"serve\", with two properties\n CREATE EDGE serve(start_year int, end_year int);\nIf the preceding statements are executed successfully, the schema is created. You can run the statements as follows to view the schema.\n// To list all the tags in the current graph space\n SHOW TAGS;\n// To list all the edge types in the current graph space\n SHOW EDGES;\n// To view the definition of the tags and edge types\nDESCRIBE TAG player;\nDESCRIBE TAG team;\nDESCRIBE EDGE follow;\nDESCRIBE EDGE serve;\nIf the schema is created successfully, in the result window, you can see the definition of the tags and edge types.\nNext to do\nWhen a schema is created, you can import data.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/quick-start/st-ug-create-schema/","type":"doc"},{"title":"Import data","content":"Studio supports importing data in CSV format into NebulaGraph through an interface.\nPrerequisites\nTo batch import data, do a check of these:\nThe schema has been created in NebulaGraph.\nThe CSV files meet the demands of the schema.\nThe account has GOD, ADMIN, or DBA permissions. For details, see Built-in Roles.\nEntry\nIn the top navigation bar, click .\nImporting data is divided into 2 parts, creating a new data source and creating an import task, which will be described in detail next.\nCreate a new data source\nClick New Data Source in the upper right corner of the page to set the data source and its related settings. Currently, 3 types of data sources are supported.\nType of data source\nDescription\nCloud storage\nAdd cloud storage as the CSV file source, which only supports cloud services compatible with the Amazon S3 interface.\nSFTP\nAdd SFTP as the CSV file source.\nLocal file\nUpload a local CSV file. The file size can not exceed 200 MB, please put the files exceeding the limit into other types of data sources.\nCreate an import task\nClick New Import at the top left corner of the page to complete the following settings:\nSpace: The name of the graph space where the data needs to be imported.\nTask Name: automatically generated by default, can be modified.\n(optional)More configuration: You can customize the concurrency, batch size, retry times, read concurrency, and import concurrency.\nMap Tags:\nClick Add Tag, and then select the tag within the added tags below.\nClick Add source file, select Data Source Type and File Path in Data source file, find the file you need to import, and then click Add.\nIn the preview page, set the file separator and whether to carry the table header, and then click Confirm.\nSelect the corresponding column for VID in VID Columns. You can select multiple columns to be merged into a VID, and you can also add a prefix or suffix to the VID.\nSelect the corresponding column for the attribute in the properties box. For properties that can be NULL or have DEFAULT set, you can leave the corresponding column unspecified.\nRepeat steps 2 to 5 to import all the data files of the Tag selected.\nRepeat steps 1 to 6 to import all Tag data.\nMap Edges: Same operation as map tags.\nAfter completing the settings, click Import, enter the password for the NebulaGraph account, and confirm.\nAfter the import task is created, you can view the progress of the import task in the Import Data tab, which supports operations such as filtering tasks based on graph space, editing the task, viewing logs, downloading logs, reimporting, downloading configuration files, and deleting tasks.\nNext\nAfter completing the data import, users can access the Console page.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/quick-start/st-ug-import-data/","type":"doc"},{"title":"Console","content":"Studio console interface is shown as follows.\nThe following table lists various functions on the console interface.\nnumber\nfunction\ndescriptions\ntoolbar\nClick the Console tab to enter the console page.\nselect a space\nSelect a space in the Current Graph Space list. descriptions: Studio does not support running the USE statements directly in the input box.\nfavorites\nClick the button to expand the favorites, click one of the statements, and the input box will automatically enter the statement.\nhistory list\nClick button representing the statement record. In the statement running record list, click one of the statements, and the statement will be automatically entered in the input box. The list provides the record of the last 15 statements.\nclean input box\nClick button to clear the content entered in the input box.\nrun\nAfter inputting the nGQL statement in the input box, click button to indicate the operation to start running the statement.\ncustom parameters display\nClick the button to expand the custom parameters for parameterized query. For details, see Manage parameters.\ninput box\nAfter inputting the nGQL statements, click the button to run the statement. You can input multiple statements and run them at the same time by using the separator ;, and also use the symbol // to add comments.\nstatement running status\nAfter running the nGQL statement, the statement running status is displayed. If the statement runs successfully, the statement is displayed in green. If the statement fails, the statement is displayed in red.\n10\nadd to favorites\nClick the button to save the statement as a favorite, the button for the favorite statement is colored in yellow exhibit.\n11\nexport CSV file or PNG file\nAfter running the nGQL statement to return the result, when the result is in Table window, click the button to export as a CSV file. Switch to the Graph window and click the button to save the results as a CSV file or PNG image export.\n12\nexpand/hide execution results\nClick the button to hide the result or click button to expand the result.\n13\nclose execution results\nClick the button to close the result returned by this nGQL statement.\n14\nTable window\nDisplay the result from running nGQL statement. If the statement returns results, the window displays the results in a table.\n15\nGraph window\nDisplay the result from running nGQL statement. If the statement returns the complete vertex-edge result, the window displays the result as a graph . Click the button on the right to view the overview panel.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/quick-start/st-ug-console/","type":"doc"},{"title":"Manage graph spaces","content":"When Studio is connected to NebulaGraph, you can create or delete a graph space. You can use the Console page or the Schema page to do these operations. This article only introduces how to use the Schema page to operate graph spaces in NebulaGraph.\nPrerequisites\nTo operate a graph space on the Schema page of Studio, you must do a check of these:\nStudio is connected to NebulaGraph.\nYour account has the authority of GOD. It means that:\nIf the authentication is enabled in NebulaGraph, you can use root and any password to sign in to Studio.\nIf the authentication is disabled in NebulaGraph, you must use root and its password to sign in to Studio.\nCreate a graph space\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, click Create Space, do these settings:\nName: Specify a name to the new graph space. In this example, basketballplayer is used. The name must be unique in the database.\nVid Type: The data types of VIDs are restricted to FIXED_STRING() or INT64. A graph space can only select one VID type. In this example, FIXED_STRING(32) is used. For more information, see VID.\nComment: Enter the description for graph space. The maximum length is 256 bytes. By default, there will be no comments on a space. But in this example, Statistics of basketball players is used.\nOptional Parameters: Set the values of partition_num and replica_factor respectively. In this example, these parameters are set to 100 and 1 respectively. For more information, see CREATE SPACE syntax.\nIn the Equivalent to the following nGQL statement panel, you can see the statement equivalent to the preceding settings.\nCREATE SPACE basketballplayer (partition_num = 100, replica_factor = 1, vid_type = FIXED_STRING(32)) COMMENT = \"Statistics of basketball players\"\nConfirm the settings and then click the + Create button. If the graph space is created successfully, you can see it on the graph space list.\nDelete a graph space\nIn the toolbar, click the Schema tab.\nIn the Graph Space List, find the space you want to be deleted, and click Delete Graph Space in the Operation column.\nOn the dialog box, confirm the information and then click OK. \nNext to do\nAfter a graph space is created, you can create or edit a schema, including:\nOperate tags\nOperate edge types\nOperate indexes","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/manage-schema/st-ug-crud-space/","type":"doc"},{"title":"Manage tags","content":"After a graph space is created in NebulaGraph, you can create tags. With Studio, you can use the Console page or the Schema page to create, retrieve, update, or delete tags. This topic introduces how to use the Schema page to operate tags in a graph space only.\nPrerequisites\nTo operate a tag on the Schema page of Studio, you must do a check of these:\nStudio is connected to NebulaGraph.\nA graph space is created.\nYour account has the authority of GOD, ADMIN, or DBA.\nCreate a tag\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Tag tab and click the + Create button.\nOn the Create page, do these settings:\nName: Specify an appropriate name for the tag. In this example, course is specified.\nComment (Optional): Enter the description for tag.\nDefine Properties (Optional): If necessary, click + Add Property to do these settings:\nEnter a property name.\nSelect a data type.\nSelect whether to allow null values..\n(Optional) Enter the default value.\n(Optional) Enter the description.\nSet TTL (Time To Live) (Optional): If no index is set for the tag, you can set the TTL configuration: In the upper left corner of the Set TTL panel, click the check box to expand the panel, and configure TTL_COL and TTL_ DURATION (in seconds). For more information about both parameters, see TTL configuration.\nWhen the preceding settings are completed, in the Equivalent to the following nGQL statement panel, you can see the nGQL statement equivalent to these settings.\nConfirm the settings and then click the + Create button. \nWhen the tag is created successfully, the Define Properties panel shows all its properties on the list.\nEdit a tag\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Tag tab, find a tag and then click the button in the Operations column.\nOn the Edit page, do these operations:\nTo edit a Comment: Click Edit on the right of Comment.\nTo edit a property: On the Define Properties panel, find a property, click Edit, and then change the data type or the default value.\nTo delete a property: On the Define Properties panel, find a property, click Delete.\nTo add more properties: On the Define Properties panel, click the Add Property button to add a new property.\nTo set the TTL configuration: In the upper left corner of the Set TTL panel, click the check box and then set TTL.\nTo delete the TTL configuration: When the Set TTL panel is expanded, in the upper left corner of the panel, click the check box to delete the configuration.\nTo edit the TTL configuration: On the Set TTL panel, click Edit and then change the configuration of TTL_COL and TTL_DURATION (in seconds).\nDelete a tag\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Tag tab, find an tag and then click the button in the Operations column.\nClick OK to confirm delete a tag in the pop-up dialog box.\nNext to do\nAfter the tag is created, you can use the Console page to insert vertex data one by one manually or use the Import page to bulk import vertex data.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/manage-schema/st-ug-crud-tag/","type":"doc"},{"title":"Manage edge types","content":"After a graph space is created in NebulaGraph, you can create edge types. With Studio, you can choose to use the Console page or the Schema page to create, retrieve, update, or delete edge types. This topic introduces how to use the Schema page to operate edge types in a graph space only.\nPrerequisites\nTo operate an edge type on the Schema page of Studio, you must do a check of these:\nStudio is connected to NebulaGraph.\nA graph space is created.\nYour account has the authority of GOD, ADMIN, or DBA.\nCreate an edge type\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Edge Type tab and click the + Create button.\nOn the Create Edge Type page, do these settings:\nName: Specify an appropriate name for the edge type. In this example, serve is used.\nComment (Optional): Enter the description for edge type.\nDefine Properties (Optional): If necessary, click + Add Property to do these settings:\nEnter a property name.\nSelect a data type.\nSelect whether to allow null values..\n(Optional) Enter the default value.\n(Optional) Enter the description.\nSet TTL (Time To Live) (Optional): If no index is set for the edge type, you can set the TTL configuration: In the upper left corner of the Set TTL panel, click the check box to expand the panel, and configure TTL_COL and TTL_ DURATION (in seconds). For more information about both parameters, see TTL configuration.\nWhen the preceding settings are completed, in the Equivalent to the following nGQL statement panel, you can see the nGQL statement equivalent to these settings.\nConfirm the settings and then click the + Create button.\nWhen the edge type is created successfully, the Define Properties panel shows all its properties on the list.\nEdit an edge type\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Edge Type tab, find an edge type and then click the button in the Operations column.\nOn the Edit page, do these operations:\nTo edit a comment: Click Edit on the right of Comment.\nTo edit a property: On the Define Properties panel, find a property, click Edit, and then change the data type or the default value.\nTo delete a property: On the Define Properties panel, find a property, click Delete.\nTo add more properties: On the Define Properties panel, click the Add Property button to add a new property.\nTo set the TTL configuration: In the upper left corner of the Set TTL panel, click the check box and then set TTL.\nTo delete the TTL configuration: When the Set TTL panel is expanded, in the upper left corner of the panel, click the check box to delete the configuration.\nTo edit the TTL configuration: On the Set TTL panel, click Edit and then change the configuration of TTL_COL and TTL_DURATION (in seconds).\nDelete an Edge type\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Edge Type tab, find an edge type and then click the button in the Operations column.\nClick OK to confirm in the pop-up dialog box.\nNext to do\nAfter the edge type is created, you can use the Console page to insert edge data one by one manually or use the Import page to bulk import edge data.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/manage-schema/st-ug-crud-edge-type/","type":"doc"},{"title":"Manage indexes","content":"You can create an index for a Tag and/or an Edge type. An index lets traversal start from vertices or edges with the same property and it can make a query more efficient. With Studio, you can use the Console page or the Schema page to create, retrieve, and delete indexes. This topic introduces how to use the Schema page to operate an index only.\nPrerequisites\nTo operate an index on the Schema page of Studio, you must do a check of these:\nStudio is connected to NebulaGraph.\nA graph Space, Tags, and Edge Types are created.\nYour account has the authority of GOD, ADMIN, or DBA.\nCreate an index\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Index tab and then click the + Create button.\nOn the Create page, do these settings:\nIndex Type: Choose to create an index for a tag or for an edge type. In this example, Edge Type is chosen.\nAssociated tag name: Choose a tag name or an edge type name. In this example, follow is chosen.\nIndex Name: Specify a name for the new index. In this example, follow_index is used.\nComment (Optional): Enter the description for index.\nIndexed Properties (Optional): Click Add property, and then, in the dialog box, choose a property. If necessary, repeat this step to choose more properties. You can drag the properties to sort them. In this example, degree is chosen.\nWhen the settings are done, the Equivalent to the following nGQL statement panel shows the statement equivalent to the settings. \nConfirm the settings and then click the + Create button. When an index is created, the index list shows the new index.\nView indexes\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Index tab, in the upper left corner, choose an index type, Tag or Edge Type.\nIn the list, find an index and click its row. All its details are shown in the expanded row.\nRebuild indexes\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Index tab, in the upper left corner, choose an index type, Tag or Edge Type.\nClick the Index tab, find an index and then click the button Rebuild in the Operations column.\nDelete an index\nTo delete an index on Schema, follow these steps:\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nIn the Current Graph Space field, confirm the name of the graph space. If necessary, you can choose another name to change the graph space.\nClick the Index tab, find an index and then click the button in the Operations column.\nClick OK to confirm in the pop-up dialog box.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/manage-schema/st-ug-crud-index/","type":"doc"},{"title":"View Schema","content":"Users can visually view schemas in NebulaGraph Studio.\nSteps\nIn the toolbar, click the Schema tab.\nIn the Graph Space List page, find a graph space and then click its name or click Schema in the Operations column.\nClick View Schema tab and click the Get Schema button.\nOther operations\nIn the Graph Space List page, find a graph space and then perform the following operations in the Operations column:\nView Schema DDL: Displays schema creation statements for the graph space, including graph spaces, tags, edge types, and indexes.\nClone Graph Space: Clones the schema of the graph space to a new graph space.\nDelete Graph pace: Deletes the graph space, including the schema and all vertices and edges.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/manage-schema/st-ug-view-schema/","type":"doc"},{"title":"Schema drafting","content":"Studio supports the schema drafting function. Users can design their schemas on the canvas to visually display the relationships between vertices and edges, and apply the schema to a specified graph space after the design is completed.\nFeatures\nDesign schema visually.\nApplies schema to a specified graph space.\nExport the schema as a PNG image.\nEntry\nAt the top navigation bar, click .\nDesign schema\nThe following steps take designing the schema of the basketballplayer dataset as an example to demonstrate how to use the schema drafting function.\nAt the upper left corner of the page, click New.\nCreate a tag by selecting the appropriate color tag under the canvas. You can hold down the left button and drag the tag into the canvas.\nClick the tag. On the right side of the page, you need to fill in the name of the tag as player, and add two properties name and age.\nCreate a tag again. The name of the tag is team, and the property is name.\nConnect from the anchor point of the tag player to the anchor point of the tag team. Click the generated edge, fill in the name of the edge type as serve, and add two properties start_year and end_year.\nConnect from an anchor point of the tag player to another one of its own. Click the generated edge, fill in the name of the edge type as follow, and add a property degree.\nAfter the design is complete, click at the top of the page to change the name of the draft, and then click at the top right corner to save the draft.\nApply schema\nSelect the draft that you want to import from the Draft list on the left side of the page, and then click Apply to Space at the upper right corner.\nImport the schema to a new or existing space, and click Confirm.\nModify schema\nSelect the schema draft that you want to modify from the Draft list on the left side of the page. Click at the upper right corner after the modification.\nDelete schema\nSelect the schema draft that you want to delete from the Draft list on the left side of the page, click X at the upper right corner of the thumbnail, and confirm to delete it.\nExport Schema\nClick at the upper right corner to export the schema as a PNG image.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/quick-start/draft/","type":"doc"},{"title":"Connecting to the database error","content":"Problem description\nAccording to the connect Studio operation, it prompts failed.\nPossible causes and solutions\nYou can troubleshoot the problem by following the steps below.\nStep1: Confirm that the format of the Host field is correct\nYou must fill in the IP address (graph_server_ip) and port of the NebulaGraph database Graph service. If no changes are made, the port defaults to 9669. Even if NebulaGraph and Studio are deployed on the current machine, you must use the local IP address instead of 127.0.0.1, localhost or 0.0.0.0.\nStep2: Confirm that the username and password are correct\nIf authentication is not enabled, you can use root and any password as the username and its password.\nIf authentication is enabled and different users are created and assigned roles, users in different roles log in with their accounts and passwords.\nStep3: Confirm that NebulaGraph service is normal\nCheck NebulaGraph service status. Regarding the operation of viewing services:\nIf you compile and deploy NebulaGraph on a Linux server, refer to the NebulaGraph service.\nIf you use NebulaGraph deployed by Docker Compose and RPM, refer to the NebulaGraph service status and ports.\nIf the NebulaGraph service is normal, proceed to Step 4 to continue troubleshooting. Otherwise, please restart NebulaGraph service.\nStep4: Confirm the network connection of the Graph service is normal\nRun a command (for example, telnet 9669) on the Studio machine to confirm whether NebulaGraph's Graph service network connection is normal.\nIf the connection fails, check according to the following steps:\nIf Studio and NebulaGraph are on the same machine, check if the port is exposed.\nIf Studio and NebulaGraph are not on the same machine, check the network configuration of the NebulaGraph server, such as firewall, gateway, and port.\nIf you cannot connect to the NebulaGraph service after troubleshooting with the above steps, please go to the NebulaGraph forum for consultation.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/troubleshooting/st-ug-config-server-errors/","type":"doc"},{"title":"Cannot access to Studio","content":"Problem description\nI follow the document description and visit 127.0.0.1:7001 or 0.0.0.0:7001 after starting Studio, why can’t I open the page?\nPossible causes and solutions\nYou can troubleshoot the problem by following the steps below.\nStep1: Confirm system architecture\nIt is necessary to confirm whether the machine where the Studio service is deployed is of x86_64 architecture. Currently, Studio only supports x86_64 architecture.\nStep2: Check if the Studio service starts normally\nFor Studio deployed with RPM or DEB packages, use systemctl status nebula-graph-studio to see the running status.\nFor Studio deployed with tar package, use sudo lsof -i:7001 to check port status.\nFor Studio deployed with docker, use docker-compose ps to see the running status.\nRun docker-compose ps to check if the service has started normally.\nIf the service is normal, the return result is as follows. Among them, the State column should all be displayed as Up.\n Name Command State Ports\n------------------------------------------------------------------------------------------------------\nnebula-web-docker_client_1 ./nebula-go-api Up 0.0.0.0:32782->8080/tcp\nnebula-web-docker_importer_1 nebula-importer --port=569 ... Up 0.0.0.0:32783->5699/tcp\nnebula-web-docker_nginx_1 /docker-entrypoint.sh ngin ... Up 0.0.0.0:7001->7001/tcp, 80/tcp\nnebula-web-docker_web_1 docker-entrypoint.sh npm r ... Up 0.0.0.0:32784->7001/tcp\nIf the above result is not returned, stop Studio and restart it first. For details, refer to Deploy Studio.\n!!! note\n If you used `docker-compose up -d` to satrt NebulaGraph before, you must run the `docker-compose down` to stop NebulaGraph.\nStep3: Confirm address\nIf Studio and the browser are on the same machine, users can use localhost:7001, 127.0.0.1:7001 or 0.0.0.0:7001 in the browser to access Studio.\nIf Studio and the browser are not on the same machine, you must enter :7001 in the browser. Among them, studio_server_ip refers to the IP address of the machine where the Studio service is deployed.\nStep4: Confirm network connection\nRun curl :7001 -I to confirm if it is normal. If it returns HTTP/1.1 200 OK, it means that the network is connected normally.\nIf the connection is refused, check according to the following steps:\nIf the connection fails, check according to the following steps:\nIf Studio and NebulaGraph are on the same machine, check if the port is exposed.\nIf Studio and NebulaGraph are not on the same machine, check the network configuration of the NebulaGraph server, such as firewall, gateway, and port.\nIf you cannot connect to the NebulaGraph service after troubleshooting with the above steps, please go to the NebulaGraph forum for consultation.","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/troubleshooting/st-ug-connection-errors/","type":"doc"},{"title":"FAQ","content":"","url":"https://docs.nebula-graph.io/3.6.0/nebula-studio/troubleshooting/st-ug-faq/","type":"doc"},{"title":"What is NebulaGraph Dashboard Community Edition","content":"NebulaGraph Dashboard Community Edition (Dashboard for short) is a visualization tool that monitors the status of machines and services in NebulaGraph clusters.\nFeatures\nDashboard monitors:\nThe status of all the machines in clusters, including CPU, memory, load, disk, and network.\nThe information of all the services in clusters, including the IP addresses, versions, and monitoring metrics (such as the number of queries, the latency of queries, the latency of heartbeats, and so on).\nThe information of clusters, including the information of services, partitions, configurations, and long-term tasks.\nSet how often the metrics page refreshes.\nScenarios\nYou can use Dashboard in one of the following scenarios:\nYou want to monitor key metrics conveniently and quickly, and present multiple key information of the business to ensure the business operates normally.\nYou want to monitor clusters from multiple dimensions (such as the time, aggregate rules, and metrics).\nAfter a failure occurs, you need to review it and confirm its occurrence time and unexpected phenomena.\nPrecautions\nThe monitoring data will be retained for 14 days by default, that is, only the monitoring data within the last 14 days can be queried.\nVersion compatibility\nThe version correspondence between NebulaGraph and Dashboard Community Edition is as follows.\nNebulaGraph version\nDashboard version\n3.6.0\n3.4.0\n3.5.x\n3.4.0\n3.4.0 ~ 3.4.1\n3.4.0、3.2.0\n3.3.0\n3.2.0\n2.5.0 ~ 3.2.0\n3.1.0\n2.5.x ~ 3.1.0\n1.1.1\n2.0.1~2.5.1\n1.0.2\n2.0.1~2.5.1\n1.0.1\nRelease note\nRelease","url":"https://docs.nebula-graph.io/3.6.0/nebula-dashboard/1.what-is-dashboard/","type":"doc"},{"title":"Deploy Dashboard Community Edition","content":"This topic will describe how to deploy NebulaGraph Dashboard in detail.\nTo download and compile the latest source code of Dashboard, follow the instructions on the nebula dashboard GitHub page.\nPrerequisites\nBefore you deploy Dashboard, you must confirm that:\nThe NebulaGraph services are deployed and started. For more information, see NebulaGraph Database Manual.\nBefore the installation starts, the following ports are not occupied.\n9200\n9100\n9090\n8090\n7003\nThe node-exporter is installed on the machines to be monitored. For details on installation, see Prometheus document.\nSteps\nDownload the tar packagenebula-dashboard-3.4.0.x86_64.tar.gz as needed.\nRun tar -xvf nebula-dashboard-3.4.0.x86_64.tar.gz to decompress the installation package.\nModify the config.yaml file in nebula-dashboard.\nThe configuration file contains the configurations of four dependent services and configurations of clusters. The descriptions of the dependent services are as follows.\nService\nDefault port\nDescription\nnebula-http-gateway\n8090\nProvides HTTP ports for cluster services to execute nGQL statements to interact with the NebulaGraph database.\nnebula-stats-exporter\n9200\nCollects the performance metrics in the cluster, including the IP addresses, versions, and monitoring metrics (such as the number of queries, the latency of queries, the latency of heartbeats, and so on).\nnode-exporter\n9100\nCollects the source information of nodes in the cluster, including the CPU, memory, load, disk, and network.\nprometheus\n9090\nThe time series database that stores monitoring data.\nThe descriptions of the configuration file are as follows.\nport: 7003 # Web service port.\ngateway:\n ip: hostIP # The IP of the machine where the Dashboard is deployed.\n port: 8090\n https: false # Whether to enable HTTPS.\n runmode: dev # Program running mode, including dev, test, and prod. It is used to distinguish between different running environments generally.\nstats-exporter:\n ip: hostIP # The IP of the machine where the Dashboard is deployed.\n nebulaPort: 9200\n https: false # Whether to enable HTTPS.\nnode-exporter:\n - ip: nebulaHostIP_1 # The IP of the machine where the NebulaGraph is deployed.\n port: 9100\n https: false # Whether to enable HTTPS.\n# - ip: nebulaHostIP_2\n# port: 9100\n# https: false\nprometheus:\n ip: hostIP # The IP of the machine where the Dashboard is deployed.\n prometheusPort: 9090\n https: false # Whether to enable HTTPS.\n scrape_interval: 5s # The interval for collecting the monitoring data, which is 1 minute by default.\n evaluation_interval: 5s # The interval for running alert rules, which is 1 minute by default.\n# Cluster node info\nnebula-cluster:\n name: 'default' # Cluster name\n metad:\n - name: metad0\n endpointIP: nebulaMetadIP # The IP of the machine where the Meta service is deployed.\n port: 9559\n endpointPort: 19559\n # - name: metad1\n # endpointIP: nebulaMetadIP\n # port: 9559\n # endpointPort: 19559 \n graphd:\n - name: graphd0\n endpointIP: nebulaGraphdIP # The IP of the machine where the Graph service is deployed.\n port: 9669\n endpointPort: 19669\n # - name: graphd1\n # endpointIP: nebulaGraphdIP\n # port: 9669\n # endpointPort: 19669 \n storaged:\n - name: storaged0\n endpointIP: nebulaStoragedIP # The IP of the machine where the Storage service is deployed.\n port: 9779\n endpointPort: 19779\n # - name: storaged1\n # endpointIP: nebulaStoragedIP\n # port: 9779\n # endpointPort: 19779 \nRun ./dashboard.service start all to start the services.\nDeploy Dashboard with Docker Compose\nIf you are deploying Dashboard using docker, you should also modify the configuration file config.yaml, and then run docker-compose up -d to start the container.\nRun docker-compose stop to stop the container.\nManage services in Dashboard\nYou can use the dashboard.service script to start, restart, stop, and check the Dashboard services.\nsudo /dashboard.service\n[-v] [-h]\n \nParameter\nDescription\ndashboard_path\nDashboard installation path.\n-v\nDisplay detailed debugging information.\n-h\nDisplay help information.\nstart\nStart the target services.\nrestart\nRestart the target services.\nstop\nStop the target services.\nstatus\nCheck the status of the target services.\nprometheus\nSet the prometheus service as the target service.\nwebserver\nSet the webserver Service as the target service.\nexporter\nSet the exporter Service as the target service.\ngateway\nSet the gateway Service as the target service.\nall\nSet all the Dashboard services as the target services.\nNext to do\nConnect to Dashboard","url":"https://docs.nebula-graph.io/3.6.0/nebula-dashboard/2.deploy-dashboard/","type":"doc"},{"title":"Connect Dashboard","content":"After Dashboard is deployed, you can log in and use Dashboard on the browser.\nPrerequisites\nThe Dashboard services are started. For more information, see Deploy Dashboard.\nWe recommend you to use the Chrome browser of the version above 89. Otherwise, there may be compatibility issues.\nProcedures\nConfirm the IP address of the machine where the Dashboard service is installed. Enter :7003 in the browser to open the login page.\nEnter the username and the passwords of the NebulaGraph database.\nIf authentication is enabled, you can log in with the created accounts.\nIf authentication is not enabled, you can only log in using root as the username and random characters as the password.\nTo enable authentication, see Authentication.\nSelect the NebulaGraph version to be used.\nClick Login.","url":"https://docs.nebula-graph.io/3.6.0/nebula-dashboard/3.connect-dashboard/","type":"doc"},{"title":"Dashboard","content":"NebulaGraph Dashboard consists of three parts: Machine, Service, and Management. This topic will describe them in detail.\nOverview\nMachine\nClick Machine->Overview to enter the machine overview page.\nOn this page, you can view the variation of CPU, Memory, Load, Disk, and Network In/Out quickly.\nBy default, you can view the monitoring data for a maximum of 14 days. You can also select a time range or quickly select the latest 1 hour, 6 hours, 12 hours, 1 day, 3 days, 7 days, or 14 days.\nBy default, you can view the monitoring data of all the instances in clusters. You can select the instances you want to view in the instance box.\nBy default, the monitoring information page will not be updated automatically. You can set the update frequency of the monitoring information page globally or click the button to update the page manually.\nTo set a base line, click the button.\nTo view the detailed monitoring information, click the button. In this example, select Load for details. The figure is as follows.\nYou can set the monitoring time range, instance, update frequency and base line.\nYou can search for or select the target metric. For details about monitoring metrics, see Metrics.\nYou can temporarily hide nodes that you do not need to view.\nYou can click the button to view the detailed monitoring information.\nService\nClick Service->Overview to enter the service overview page.\nOn this page, you can view the information of Graph, Meta, and Storage services quickly. In the upper right corner, the number of normal services and abnormal services will be displayed.\nBy default, you can view the monitoring data for a maximum of 14 days. You can also select a time range or quickly select the latest 1 hour, 6 hours, 12 hours, 1 day, 3 days, 7 days, or 14 days.\nBy default, you can view the monitoring data of all the instances in clusters. You can select the instances you want to view in the instance box.\nBy default, the monitoring information page will not be updated automatically. You can set the update frequency of the monitoring information page globally or click the button to update the page manually.\nYou can view the status of all the services in a cluster.\nTo view the detailed monitoring information, click the button. In this example, select Graph for details. The figure is as follows.\nYou can set the monitoring time range, instance, update frequency, period, aggregation and base line.\nYou can search for or select the target metric. For details of monitoring metrics, see Monitor parameter.\nYou can temporarily hide nodes that you do not need to view.\nYou can click the button to view the detailed monitoring information.\nThe Graph service supports a set of space-level metrics. For more information, see the following section Graph space.\nGraph space\nThe service monitoring page can also monitor graph space level metrics. Only when the behavior of a graph space metric is triggered, you can specify the graph space to view information about the corresponding graph space metric.\nSpace graph metrics record the information of different graph spaces separately. Currently, only the Graph service supports a set of space-level metrics.\nFor information about the space graph metrics, see Graph space.\nManagement\nOverview info\nOn the Overview Info page, you can see the information of the NebulaGraph cluster, including Storage leader distribution, Storage service details, versions and hosts information of each NebulaGraph service, and partition distribution and details.\nStorage Leader Distribution\nIn this section, the number of Leaders and the Leader distribution will be shown.\nClick the Balance Leader button in the upper right corner to distribute Leaders evenly and quickly in the NebulaGraph cluster. For details about the Leader, see Storage Service.\nClick Detail in the upper right corner to view the details of the Leader distribution.\nVersion\nIn this section, the version and host information of each NebulaGraph service will be shown. Click Detail in the upper right corner to view the details of the version and host information.\nService information\nIn this section, the information on Storage services will be shown. The parameter description is as follows:\nParameter\nDescription\nHost\nThe IP address of the host.\nPort\nThe port of the host.\nStatus\nThe host status.\nGit Info Sha\nThe commit ID of the current version.\nLeader Count\nThe number of Leaders.\nPartition Distribution\nThe distribution of partitions.\nLeader Distribution\nThe distribution of Leaders.\nClick Detail in the upper right corner to view the details of the Storage service information.\nPartition Distribution\nSelect the specified graph space in the upper left corner, you can view the distribution of partitions in the specified graph space. You can see the IP addresses and ports of all Storage services in the cluster, and the number of partitions in each Storage service.\nClick Detail in the upper right corner to view more details.\nPartition information\nIn this section, the information on partitions will be shown. Before viewing the partition information, you need to select a graph space in the upper left corner. The parameter description is as follows:\nParameter\nDescription\nPartition ID\nThe ID of the partition.\nLeader\nThe IP address and port of the leader.\nPeers\nThe IP addresses and ports of all the replicas.\nLosts\nThe IP addresses and ports of faulty replicas.\nClick Detail in the upper right corner to view details. You can also enter the partition ID into the input box in the upper right corner of the details page to filter the shown data. \nConfig\nIt shows the configuration of the NebulaGraph service. NebulaGraph Dashboard Community Edition does not support online modification of configurations for now.\nOthers\nIn the lower left corner of the page, you can:\nSign out\nSwitch between Chinese and English\nView the current Dashboard release\nView the user manual and forum\nFold the sidebar","url":"https://docs.nebula-graph.io/3.6.0/nebula-dashboard/4.use-dashboard/","type":"doc"},{"title":"Metrics","content":"This topic will describe the monitoring metrics in NebulaGraph Dashboard.\nMachine\nCPU\nParameter\nDescription\ncpu_utilization\nThe percentage of used CPU.\ncpu_idle\nThe percentage of idled CPU.\ncpu_wait\nThe percentage of CPU waiting for IO operations.\ncpu_user\nThe percentage of CPU used by users.\ncpu_system\nThe percentage of CPU used by the system.\nMemory\nParameter\nDescription\nmemory_utilization\nThe percentage of used memory.\nmemory_used\nThe memory space used (not including caches).\nmemory_free\nThe memory space available.\nLoad\nParameter\nDescription\nload_1m\nThe average load of the system in the last 1 minute.\nload_5m\nThe average load of the system in the last 5 minutes.\nload_15m\nThe average load of the system in the last 15 minutes.\nDisk\nParameter\nDescription\ndisk_used_percentage\nThe disk utilization percentage.\ndisk_used\nThe disk space used.\ndisk_free\nThe disk space available.\ndisk_readbytes\nThe number of bytes that the system reads in the disk per second.\ndisk_writebytes\nThe number of bytes that the system writes in the disk per second.\ndisk_readiops\nThe number of read queries that the disk receives per second.\ndisk_writeiops\nThe number of write queries that the disk receives per second.\ninode_utilization\nThe percentage of used inode.\nNetwork\nParameter\nDescription\nnetwork_in_rate\nThe number of bytes that the network card receives per second.\nnetwork_out_rate\nThe number of bytes that the network card sends out per second.\nnetwork_in_errs\nThe number of wrong bytes that the network card receives per second.\nnetwork_out_errs\nThe number of wrong bytes that the network card sends out per second.\nnetwork_in_packets\nThe number of data packages that the network card receives per second.\nnetwork_out_packets\nThe number of data packages that the network card sends out per second.\nService\nPeriod\nThe period is the time range of counting metrics. It currently supports 5 seconds, 60 seconds, 600 seconds, and 3600 seconds, which respectively represent the last 5 seconds, the last 1 minute, the last 10 minutes, and the last 1 hour.\nMetric methods\nParameter\nDescription\nrate\nThe average rate of operations per second in a period.\nsum\nThe sum of operations in the period.\navg\nThe average latency in the cycle.\nP75\nThe 75th percentile latency.\nP95\nThe 95th percentile latency.\nP99\nThe 99th percentile latency.\nP999\nThe 99.9th percentile latency.\nGraph\nParameter\nDescription\nnum_active_queries\nThe number of changes in the number of active queries. Formula: The number of started queries minus the number of finished queries within a specified time.\nnum_active_sessions\nThe number of changes in the number of active sessions. Formula: The number of logged in sessions minus the number of logged out sessions within a specified time.For example, when querying num_active_sessions.sum.5, if there were 10 sessions logged in and 30 sessions logged out in the last 5 seconds, the value of this metric is -20 (10-30).\nnum_aggregate_executors\nThe number of executions for the Aggregation operator.\nnum_auth_failed_sessions_bad_username_password\nThe number of sessions where authentication failed due to incorrect username and password.\nnum_auth_failed_sessions_out_of_max_allowed\nThe number of sessions that failed to authenticate logins because the value of the parameter FLAG_OUT_OF_MAX_ALLOWED_CONNECTIONS was exceeded.\nnum_auth_failed_sessions\nThe number of sessions in which login authentication failed.\nnum_indexscan_executors\nThe number of executions for index scan operators.\nnum_killed_queries\nThe number of killed queries.\nnum_opened_sessions\nThe number of sessions connected to the server.\nnum_queries\nThe number of queries.\nnum_query_errors_leader_changes\nThe number of the raft leader changes due to query errors.\nnum_query_errors\nThe number of query errors.\nnum_reclaimed_expired_sessions\nThe number of expired sessions actively reclaimed by the server.\nnum_rpc_sent_to_metad_failed\nThe number of failed RPC requests that the Graphd service sent to the Metad service.\nnum_rpc_sent_to_metad\nThe number of RPC requests that the Graphd service sent to the Metad service.\nnum_rpc_sent_to_storaged_failed\nThe number of failed RPC requests that the Graphd service sent to the Storaged service.\nnum_rpc_sent_to_storaged\nThe number of RPC requests that the Graphd service sent to the Storaged service.\nnum_sentences\nThe number of statements received by the Graphd service.\nnum_slow_queries\nThe number of slow queries.\nnum_sort_executors\nThe number of executions for the Sort operator.\noptimizer_latency_us\nThe latency of executing optimizer statements.\nquery_latency_us\nThe latency of queries.\nslow_query_latency_us\nThe latency of slow queries.\nnum_queries_hit_memory_watermark\nThe number of queries reached the memory watermark.\nresp_part_completeness\nThe completeness of the partial success. You need to set accept_partial_success to true in the graph configuration first.\nMeta\nParameter\nDescription\ncommit_log_latency_us\nThe latency of committing logs in Raft.\ncommit_snapshot_latency_us\nThe latency of committing snapshots in Raft.\nheartbeat_latency_us\nThe latency of heartbeats.\nnum_heartbeats\nThe number of heartbeats.\nnum_raft_votes\nThe number of votes in Raft.\ntransfer_leader_latency_us\nThe latency of transferring the raft leader.\nnum_agent_heartbeats\nThe number of heartbeats for the AgentHBProcessor.\nagent_heartbeat_latency_us\nThe latency of the AgentHBProcessor.\nreplicate_log_latency_us\nThe latency of replicating the log record to most nodes by Raft.\nnum_send_snapshot\nThe number of times that Raft sends snapshots to other nodes.\nappend_log_latency_us\nThe latency of replicating the log record to a single node by Raft.\nappend_wal_latency_us\nThe Raft write latency for a single WAL.\nnum_grant_votes\nThe number of times that Raft votes for other nodes.\nnum_start_elect\nThe number of times that Raft starts an election.\nStorage\nParameter\nDescription\nadd_edges_latency_us\nThe latency of adding edges.\nadd_vertices_latency_us\nThe latency of adding vertices.\ncommit_log_latency_us\nThe latency of committing logs in Raft.\ncommit_snapshot_latency_us\nThe latency of committing snapshots in Raft.\ndelete_edges_latency_us\nThe latency of deleting edges.\ndelete_vertices_latency_us\nThe latency of deleting vertices.\nget_neighbors_latency_us\nThe latency of querying neighbor vertices.\nget_dst_by_src_latency_us\nThe latency of querying the destination vertex by the source vertex.\nnum_get_prop\nThe number of executions for the GetPropProcessor.\nnum_get_neighbors_errors\nThe number of execution errors for the GetNeighborsProcessor.\nnum_get_dst_by_src_errors\nThe number of execution errors for the GetDstBySrcProcessor.\nget_prop_latency_us\nThe latency of executions for the GetPropProcessor.\nnum_edges_deleted\nThe number of deleted edges.\nnum_edges_inserted\nThe number of inserted edges.\nnum_raft_votes\nThe number of votes in Raft.\nnum_rpc_sent_to_metad_failed\nThe number of failed RPC requests that the Storage service sent to the Meta service.\nnum_rpc_sent_to_metad\nThe number of RPC requests that the Storaged service sent to the Metad service.\nnum_tags_deleted\nThe number of deleted tags.\nnum_vertices_deleted\nThe number of deleted vertices.\nnum_vertices_inserted\nThe number of inserted vertices.\ntransfer_leader_latency_us\nThe latency of transferring the raft leader.\nlookup_latency_us\nThe latency of executions for the LookupProcessor.\nnum_lookup_errors\nThe number of execution errors for the LookupProcessor.\nnum_scan_vertex\nThe number of executions for the ScanVertexProcessor.\nnum_scan_vertex_errors\nThe number of execution errors for the ScanVertexProcessor.\nupdate_edge_latency_us\nThe latency of executions for the UpdateEdgeProcessor.\nnum_update_vertex\nThe number of executions for the UpdateVertexProcessor.\nnum_update_vertex_errors\nThe number of execution errors for the UpdateVertexProcessor.\nkv_get_latency_us\nThe latency of executions for the Getprocessor.\nkv_put_latency_us\nThe latency of executions for the PutProcessor.\nkv_remove_latency_us\nThe latency of executions for the RemoveProcessor.\nnum_kv_get_errors\nThe number of execution errors for the GetProcessor.\nnum_kv_get\nThe number of executions for the GetProcessor.\nnum_kv_put_errors\nThe number of execution errors for the PutProcessor.\nnum_kv_put\nThe number of executions for the PutProcessor.\nnum_kv_remove_errors\nThe number of execution errors for the RemoveProcessor.\nnum_kv_remove\nThe number of executions for the RemoveProcessor.\nforward_tranx_latency_us\nThe latency of transmission.\nscan_edge_latency_us\nThe latency of executions for the ScanEdgeProcessor.\nnum_scan_edge_errors\nThe number of execution errors for the ScanEdgeProcessor.\nnum_scan_edge\nThe number of executions for the ScanEdgeProcessor.\nscan_vertex_latency_us\nThe latency of executions for the ScanVertexProcessor.\nnum_add_edges\nThe number of times that edges are added.\nnum_add_edges_errors\nThe number of errors when adding edges.\nnum_add_vertices\nThe number of times that vertices are added.\nnum_start_elect\nThe number of times that Raft starts an election.\nnum_add_vertices_errors\nThe number of errors when adding vertices.\nnum_delete_vertices_errors\nThe number of errors when deleting vertices.\nappend_log_latency_us\nThe latency of replicating the log record to a single node by Raft.\nnum_grant_votes\nThe number of times that Raft votes for other nodes.\nreplicate_log_latency_us\nThe latency of replicating the log record to most nodes by Raft.\nnum_delete_tags\nThe number of times that tags are deleted.\nnum_delete_tags_errors\nThe number of errors when deleting tags.\nnum_delete_edges\nThe number of edge deletions.\nnum_delete_edges_errors\nThe number of errors when deleting edges\nnum_send_snapshot\nThe number of times that snapshots are sent.\nupdate_vertex_latency_us\nThe latency of executions for the UpdateVertexProcessor.\nappend_wal_latency_us\nThe Raft write latency for a single WAL.\nnum_update_edge\nThe number of executions for the UpdateEdgeProcessor.\ndelete_tags_latency_us\nThe latency of deleting tags.\nnum_update_edge_errors\nThe number of execution errors for the UpdateEdgeProcessor.\nnum_get_neighbors\nThe number of executions for the GetNeighborsProcessor.\nnum_get_dst_by_src\nThe number of executions for the GetDstBySrcProcessor.\nnum_get_prop_errors\nThe number of execution errors for the GetPropProcessor.\nnum_delete_vertices\nThe number of times that vertices are deleted.\nnum_lookup\nThe number of executions for the LookupProcessor.\nnum_sync_data\nThe number of times the Storage service synchronizes data from the Drainer.\nnum_sync_data_errors\nThe number of errors that occur when the Storage service synchronizes data from the Drainer.\nsync_data_latency_us\nThe latency of the Storage service synchronizing data from the Drainer.\nGraph space\nParameter\nDescription\nnum_active_queries\nThe number of queries currently being executed.\nnum_queries\nThe number of queries.\nnum_sentences\nThe number of statements received by the Graphd service.\noptimizer_latency_us\nThe latency of executing optimizer statements.\nquery_latency_us\nThe latency of queries.\nnum_slow_queries\nThe number of slow queries.\nnum_query_errors\nThe number of query errors.\nnum_query_errors_leader_changes\nThe number of raft leader changes due to query errors.\nnum_killed_queries\nThe number of killed queries.\nnum_aggregate_executors\nThe number of executions for the Aggregation operator.\nnum_sort_executors\nThe number of executions for the Sort operator.\nnum_indexscan_executors\nThe number of executions for index scan operators.\nnum_auth_failed_sessions_bad_username_password\nThe number of sessions where authentication failed due to incorrect username and password.\nnum_auth_failed_sessions\nThe number of sessions in which login authentication failed.\nnum_opened_sessions\nThe number of sessions connected to the server.\nnum_queries_hit_memory_watermark\nThe number of queries reached the memory watermark.\nnum_reclaimed_expired_sessions\nThe number of expired sessions actively reclaimed by the server.\nnum_rpc_sent_to_metad_failed\nThe number of failed RPC requests that the Graphd service sent to the Metad service.\nnum_rpc_sent_to_metad\nThe number of RPC requests that the Graphd service sent to the Metad service.\nnum_rpc_sent_to_storaged_failed\nThe number of failed RPC requests that the Graphd service sent to the Storaged service.\nnum_rpc_sent_to_storaged\nThe number of RPC requests that the Graphd service sent to the Storaged service.\nslow_query_latency_us\nThe latency of slow queries.","url":"https://docs.nebula-graph.io/3.6.0/nebula-dashboard/6.monitor-parameter/","type":"doc"},{"title":"What is NebulaGraph Operator","content":"Concept\nNebulaGraph Operator is a tool to automate the deployment, operation, and maintenance of NebulaGraph clusters on Kubernetes. Building upon the excellent scalability mechanism of Kubernetes, NebulaGraph introduced its operation and maintenance knowledge into the Kubernetes system, which makes NebulaGraph a real cloud-native graph database.\nHow it works\nFor resource types that do not exist within Kubernetes, you can register them by adding custom API objects. The common way is to use the CustomResourceDefinition.\nNebulaGraph Operator abstracts the deployment management of NebulaGraph clusters as a CRD. By combining multiple built-in API objects including StatefulSet, Service, and ConfigMap, the routine management and maintenance of a NebulaGraph cluster are coded as a control loop in the Kubernetes system. When a CR instance is submitted, NebulaGraph Operator drives database clusters to the final state according to the control process.\nFeatures\nThe following features are already available in NebulaGraph Operator:\nCluster deployment and deletion: NebulaGraph Operator simplifies the process of deploying and uninstalling clusters for users. NebulaGraph Operator allows you to quickly create, update, or delete a NebulaGraph cluster by simply providing the corresponding CR file. For more information, see Install NebulaGraph Clusters.\nCluster Upgrade: NebulaGraph Operator supports cluster upgrading from version 3.5.0 to version 3.6.0.\nSelf-Healing: NebulaGraph Operator calls interfaces provided by NebulaGraph clusters to dynamically sense cluster service status. Once an exception is detected, NebulaGraph Operator performs fault tolerance. For more information, see Self-Healing.\nBalance Scheduling: Based on the scheduler extension interface, the scheduler provided by NebulaGraph Operator evenly distributes Pods in a NebulaGraph cluster across all nodes.\nLimitations\nVersion limitations\nNebulaGraph Operator does not support the v1.x version of NebulaGraph. NebulaGraph Operator version and the corresponding NebulaGraph version are as follows:\nNebulaGraph\nNebulaGraph Operator\n3.5.x ~ 3.6.0\n1.5.0 ~ 1.7.x\n3.0.0 ~ 3.4.1\n1.3.0, 1.4.0 ~ 1.4.2\n3.0.0 ~ 3.3.x\n1.0.0, 1.1.0, 1.2.0\n2.5.x ~ 2.6.x\n0.9.0\n2.5.x\n0.8.0\nRelease note\nRelease","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/1.introduction-to-nebula-operator/","type":"doc"},{"title":"Install NebulaGraph Operator","content":"You can deploy NebulaGraph Operator with Helm.\nBackground\nNebulaGraph Operator automates the management of NebulaGraph clusters, and eliminates the need for you to install, scale, upgrade, and uninstall NebulaGraph clusters, which lightens the burden on managing different application versions.\nPrerequisites\nBefore installing NebulaGraph Operator, you need to install the following software and ensure the correct version of the software :\nSoftware\nRequirement\nKubernetes\n>= 1.18\nHelm\n>= 3.2.0\nCoreDNS\n>= 1.6.0\nSteps\nAdd the NebulaGraph Operator Helm repository.\nhelm repo add nebula-operator https://vesoft-inc.github.io/nebula-operator/charts\nUpdate information of available charts locally from repositories.\nhelm repo update\nFor more information about helm repo, see Helm Repo.\nCreate a namespace for NebulaGraph Operator.\nkubectl create namespace \nFor example, run the following command to create a namespace named nebula-operator-system.\nkubectl create namespace nebula-operator-system\nAll the resources of NebulaGraph Operator are deployed in this namespace.\nInstall NebulaGraph Operator.\nhelm install nebula-operator nebula-operator/nebula-operator --namespace= --version=${chart_version}\nFor example, the command to install NebulaGraph Operator of version 1.7.3 is as follows.\nhelm install nebula-operator nebula-operator/nebula-operator --namespace=nebula-operator-system --version=1.7.3\n1.7.3 is the version of the nebula-operator chart. When not specifying --version, the latest version of the nebula-operator chart is used by default. \nRun helm search repo -l nebula-operator to see chart versions.\nYou can customize the configuration items of the NebulaGraph Operator chart before running the installation command. For more information, see Customize installation defaults.\nView the information about the default-created CRD.\nkubectl get crd\nOutput:\nNAME CREATED AT\nnebulaautoscalers.autoscaling.nebula-graph.io 2023-11-01T04:16:51Z\nnebulaclusters.apps.nebula-graph.io 2023-10-12T07:55:32Z\nnebularestores.apps.nebula-graph.io 2023-02-04T23:01:00Z\nWhat's next\nCreate a NebulaGraph cluster","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/2.get-started/2.1.install-operator/","type":"doc"},{"title":"Create a NebulaGraph cluster","content":"This topic introduces how to create a NebulaGraph cluster with the following two methods:\nCreate a NebulaGraph cluster with Helm\nCreate a NebulaGraph cluster with Kubectl\nPrerequisites\nNebulaGraph Operator is installed.\nA StorageClass is created.\nCreate a NebulaGraph cluster with Helm\nAdd the NebulaGraph Operator Helm repository.\nhelm repo add nebula-operator https://vesoft-inc.github.io/nebula-operator/charts\nUpdate information of available charts locally from chart repositories.\nhelm repo update\nSet environment variables to your desired values.\nexport NEBULA_CLUSTER_NAME=nebula # The desired NebulaGraph cluster name.\nexport NEBULA_CLUSTER_NAMESPACE=nebula # The desired namespace where your NebulaGraph cluster locates.\nexport STORAGE_CLASS_NAME=fast-disks # The name of the StorageClass that has been created.\nCreate a namespace for your NebulaGraph cluster (If you have created one, skip this step).\nkubectl create namespace \"${NEBULA_CLUSTER_NAMESPACE}\"\nApply the variables to the Helm chart to create a NebulaGraph cluster.\nhelm install \"${NEBULA_CLUSTER_NAME}\" nebula-operator/nebula-cluster \\\n --set nameOverride=\"${NEBULA_CLUSTER_NAME}\" \\\n --set nebula.storageClassName=\"${STORAGE_CLASS_NAME}\" \\\n # Specify the version of the NebulaGraph cluster. \n --set nebula.version=v3.6.0 \\ \n # Specify the version of the nebula-cluster chart. If not specified, the latest version of the chart is installed by default.\n # Run 'helm search repo nebula-operator/nebula-cluster' to view the available versions of the chart. \n --version=1.7.3 \\\n --namespace=\"${NEBULA_CLUSTER_NAMESPACE}\" \\\nCreate a NebulaGraph cluster with Kubectl\nThe following example shows how to create a NebulaGraph cluster by creating a cluster named nebula.\nCreate a namespace, for example, nebula. If not specified, the default namespace is used.\nkubectl create namespace nebula\nDefine the cluster configuration file nebulacluster.yaml.\nExpand to see an example configuration for the cluster\napiVersion: apps.nebula-graph.io/v1alpha1\nkind: NebulaCluster\nmetadata:\n name: nebula\n namespace: default\nspec:\n topologySpreadConstraints:\n - topologyKey: \"kubernetes.io/hostname\"\n whenUnsatisfiable: \"ScheduleAnyway\"\n graphd:\n # Container image for the Graph service.\n image: vesoft/nebula-graphd\n logVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n # Storage class name for storing Graph service logs.\n storageClassName: local-sc\n replicas: 1\n resources:\n limits:\n cpu: \"1\"\n memory: 1Gi\n requests:\n cpu: 500m\n memory: 500Mi\n version: v3.6.0\n imagePullPolicy: Always\n metad:\n # Container image for the Meta service.\n image: vesoft/nebula-metad\n logVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: local-sc\n dataVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: local-sc\n replicas: 1\n resources:\n limits:\n cpu: \"1\"\n memory: 1Gi\n requests:\n cpu: 500m\n memory: 500Mi\n version: v3.6.0\n reference:\n name: statefulsets.apps\n version: v1\n schedulerName: default-scheduler\n storaged:\n # Container image for the Storage service.\n image: vesoft/nebula-storaged\n logVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: local-sc\n dataVolumeClaims:\n - resources:\n requests:\n storage: 2Gi\n storageClassName: local-sc\n replicas: 1\n resources:\n limits:\n cpu: \"1\"\n memory: 1Gi\n requests:\n cpu: 500m\n memory: 500Mi\n version: v3.6.0\nFor more information about the other parameters, see Install NebulaGraph clusters.\nCreate a NebulaGraph cluster.\nkubectl create -f nebulacluster.yaml\nOutput:\nnebulacluster.apps.nebula-graph.io/nebula created\nCheck the status of the NebulaGraph cluster.\nkubectl get nc nebula\nOutput:\nNAME READY GRAPHD-DESIRED GRAPHD-READY METAD-DESIRED METAD-READY STORAGED-DESIRED STORAGED-READY AGE\nnebula True 1 1 1 1 1 1 86s\nWhat's next\nConnect to a cluster","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/2.get-started/2.3.create-cluster/","type":"doc"},{"title":"Connect to a NebulaGraph cluster","content":"After creating a NebulaGraph cluster with NebulaGraph Operator on Kubernetes, you can connect to NebulaGraph databases from within the cluster and outside the cluster.\nPrerequisites\nA NebulaGraph cluster is created on Kubernetes. For more information, see Create a NebulaGraph cluster.\nConnect to NebulaGraph databases from within a NebulaGraph cluster\nYou can create a ClusterIP type Service to provide an access point to the NebulaGraph database for other Pods within the cluster. By using the Service's IP and the Graph service's port number (9669), you can connect to the NebulaGraph database. For more information, see ClusterIP.\nCreate a file named graphd-clusterip-service.yaml. The file contents are as follows:\napiVersion: v1\nkind: Service\nmetadata:\n labels:\n app.kubernetes.io/cluster: nebula\n app.kubernetes.io/component: graphd\n app.kubernetes.io/managed-by: nebula-operator\n app.kubernetes.io/name: nebula-graph\n name: nebula-graphd-svc\n namespace: default\nspec:\n ports:\n - name: thrift\n port: 9669\n protocol: TCP\n targetPort: 9669\n - name: http\n port: 19669\n protocol: TCP\n targetPort: 19669\n selector:\n app.kubernetes.io/cluster: nebula\n app.kubernetes.io/component: graphd\n app.kubernetes.io/managed-by: nebula-operator\n app.kubernetes.io/name: nebula-graph\n type: ClusterIP # Set the type to ClusterIP.\nNebulaGraph uses port 9669 by default. 19669 is the HTTP port of the Graph service in a NebulaGraph cluster.\ntargetPort is the port mapped to the database Pods, which can be customized.\nCreate a ClusterIP Service.\nkubectl create -f graphd-clusterip-service.yaml \nCheck the IP of the Service:\n$ kubectl get service -l app.kubernetes.io/cluster=< # < is the name of your NebulaGraph cluster.\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\nnebula-graphd-svc ClusterIP 10.98.213.34 9669/TCP,19669/TCP,19670/TCP 23h\n...\nRun the following command to connect to the NebulaGraph database using the IP of the -graphd-svc Service above:\nkubectl run -ti --image vesoft/nebula-console:v3.5.0 --restart=Never -- -addr -port -u -p \nFor example:\nkubectl run -ti --image vesoft/nebula-console:v3.5.0 --restart=Never -- nebula-console -addr 10.98.213.34 -port 9669 -u root -p vesoft\n--image: The image for the tool NebulaGraph Console used to connect to NebulaGraph databases.\n: The custom Pod name.\n-addr: The IP of the ClusterIP Service, used to connect to Graphd services.\n-port: The port to connect to Graphd services, the default port of which is 9669.\n-u: The username of your NebulaGraph account. Before enabling authentication, you can use any existing username. The default username is root.\n-p: The password of your NebulaGraph account. Before enabling authentication, you can use any characters as the password.\nA successful connection to the database is indicated if the following is returned:\nIf you don't see a command prompt, try pressing enter.\n(root@nebula) [(none)]>\nYou can also connect to NebulaGraph databases with Fully Qualified Domain Name (FQDN). The domain format is -graphd..svc.. The default value of CLUSTER_DOMAIN is cluster.local.\nkubectl run -ti --image vesoft/nebula-console:v3.5.0 --restart=Never -- -addr -graphd-svc.default.svc.cluster.local -port -u -p \nservice_port is the port to connect to Graphd services, the default port of which is 9669.\nFor information about the nebula-console container, see nebula-console.\nConnect to NebulaGraph databases from outside a NebulaGraph cluster via NodePort\nYou can create a NodePort type Service to access internal cluster services from outside the cluster using any node IP and the exposed node port. You can also utilize load balancing services provided by cloud vendors (such as Azure, AWS, etc.) by setting the Service type to LoadBalancer. This allows external access to internal cluster services through the public IP and port of the load balancer provided by the cloud vendor.\nThe Service of type NodePort forwards the front-end requests via the label selector spec.selector to Graphd pods with labels app.kubernetes.io/cluster: and app.kubernetes.io/component: graphd.\nAfter creating a NebulaGraph cluster based on the example template, where spec.graphd.service.type=NodePort, the NebulaGraph Operator will automatically create a NodePort type Service named -graphd-svc in the same namespace. You can directly connect to the NebulaGraph database through any node IP and the exposed node port (see step 4 below). You can also create a custom Service according to your needs.\nSteps:\nCreate a YAML file named graphd-nodeport-service.yaml. The file contents are as follows:\napiVersion: v1\nkind: Service\nmetadata:\n labels:\n app.kubernetes.io/cluster: nebula\n app.kubernetes.io/component: graphd\n app.kubernetes.io/managed-by: nebula-operator\n app.kubernetes.io/name: nebula-graph\n name: nebula-graphd-svc-nodeport\n namespace: default\nspec:\n externalTrafficPolicy: Local\n ports:\n - name: thrift\n port: 9669\n protocol: TCP\n targetPort: 9669\n - name: http\n port: 19669\n protocol: TCP\n targetPort: 19669\n selector:\n app.kubernetes.io/cluster: nebula\n app.kubernetes.io/component: graphd\n app.kubernetes.io/managed-by: nebula-operator\n app.kubernetes.io/name: nebula-graph\n type: NodePort # Set the type to NodePort.\nNebulaGraph uses port 9669 by default. 19669 is the HTTP port of the Graph service in a NebulaGraph cluster.\nThe value of targetPort is the port mapped to the database Pods, which can be customized.\nRun the following command to create a NodePort Service.\nkubectl create -f graphd-nodeport-service.yaml\nCheck the port mapped on all of your cluster nodes.\nkubectl get services -l app.kubernetes.io/cluster=< # < is the name of your NebulaGraph cluster.\nOutput:\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\nnebula-graphd-svc-nodeport NodePort 10.107.153.129 9669:32236/TCP,19669:31674/TCP,19670:31057/TCP 24h\n...\nAs you see, the mapped port of NebulaGraph databases on all cluster nodes is 32236.\nConnect to NebulaGraph databases with your node IP and the node port above.\nkubectl run -ti --image vesoft/nebula-console:v3.5.0 --restart=Never -- -addr -port -u -p \nFor example:\nkubectl run -ti --image vesoft/nebula-console:v3.5.0 --restart=Never -- nebula-console -addr 192.168.8.24 -port 32236 -u root -p vesoft\nIf you don't see a command prompt, try pressing enter.\n(root@nebula) [(none)]>\n--image: The image for the tool NebulaGraph Console used to connect to NebulaGraph databases.\n: The custom Pod name. The above example uses nebula-console.\n-addr: The IP of any node in a NebulaGraph cluster. The above example uses 192.168.8.24.\n-port: The mapped port of NebulaGraph databases on all cluster nodes. The above example uses 32236.\n-u: The username of your NebulaGraph account. Before enabling authentication, you can use any existing username. The default username is root.\n-p: The password of your NebulaGraph account. Before enabling authentication, you can use any characters as the password.\nConnect to NebulaGraph databases from outside a NebulaGraph cluster via Ingress\nWhen dealing with multiple pods in a cluster, managing services for each pod separately is not a good practice. Ingress is a Kubernetes resource that provides a unified entry point for accessing multiple services. Ingress can be used to expose multiple services under a single IP address.\nNginx Ingress is an implementation of Kubernetes Ingress. Nginx Ingress watches the Ingress resource of a Kubernetes cluster and generates the Ingress rules into Nginx configurations that enable Nginx to forward 7 layers of traffic.\nYou can use Nginx Ingress to connect to a NebulaGraph cluster from outside the cluster using a combination of the host network and DaemonSet pattern.\nDue to the use of HostNetwork, Nginx Ingress pods may be scheduled on the same node (port conflicts will occur when multiple pods try to listen on the same port on the same node). To avoid this situation, Nginx Ingress is deployed on these nodes in DaemonSet mode (ensuring that a pod replica runs on each node in the cluster). You first need to select some nodes and label them for the specific deployment of Nginx Ingress.\nIngress does not support TCP or UDP services. For this reason, the nginx-ingress-controller pod uses the flags --tcp-services-configmap and --udp-services-configmap to point to an existing ConfigMap where the key refers to the external port to be used and the value refers to the format of the service to be exposed. The format of the value is :.\nFor example, the configurations of the ConfigMap named as tcp-services is as follows:\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: tcp-services\n namespace: nginx-ingress\ndata:\n # update \n 9769: \"default/nebula-graphd-svc:9669\"\nSteps are as follows.\nCreate a file named nginx-ingress-daemonset-hostnetwork.yaml. \nClick on nginx-ingress-daemonset-hostnetwork.yaml to view the complete content of the example YAML file.\nLabel a node where the DaemonSet named nginx-ingress-controller in the above YAML file (The node used in this example is named worker2 with an IP of 192.168.8.160) runs.\nkubectl label node worker2 nginx-ingress=true\nRun the following command to enable Nginx Ingress in the cluster you created. \nkubectl create -f nginx-ingress-daemonset-hostnetwork.yaml\nOutput:\nconfigmap/nginx-ingress-controller created\nconfigmap/tcp-services created\nserviceaccount/nginx-ingress created\nserviceaccount/nginx-ingress-backend created\nclusterrole.rbac.authorization.k8s.io/nginx-ingress created\nclusterrolebinding.rbac.authorization.k8s.io/nginx-ingress created\nrole.rbac.authorization.k8s.io/nginx-ingress created\nrolebinding.rbac.authorization.k8s.io/nginx-ingress created\nservice/nginx-ingress-controller-metrics created\nservice/nginx-ingress-default-backend created\nservice/nginx-ingress-proxy-tcp created\ndaemonset.apps/nginx-ingress-controller created\nSince the network type that is configured in Nginx Ingress is hostNetwork, after successfully deploying Nginx Ingress, with the IP (192.168.8.160) of the node where Nginx Ingress is deployed and with the external port (9769) you define, you can access NebulaGraph. \nUse the IP address and the port configured in the preceding steps. You can connect to NebulaGraph with NebulaGraph Console. \nkubectl run -ti --image vesoft/nebula-console:v3.5.0 --restart=Never -- -addr -port -u -p \nOutput:\nkubectl run -ti --image vesoft/nebula-console:v3.5.0 --restart=Never -- nebula-console -addr 192.168.8.160 -port 9769 -u root -p vesoft\n--image: The image for the tool NebulaGraph Console used to connect to NebulaGraph databases.\n The custom Pod name. The above example uses nebula-console.\n-addr: The IP of the node where Nginx Ingress is deployed. The above example uses 192.168.8.160.\n-port: The port used for external network access. The above example uses 9769.\n-u: The username of your NebulaGraph account. Before enabling authentication, you can use any existing username. The default username is root.\n-p: The password of your NebulaGraph account. Before enabling authentication, you can use any characters as the password.\nA successful connection to the database is indicated if the following is returned:\nIf you don't see a command prompt, try pressing enter.\n(root@nebula) [(none)]>","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/2.get-started/2.4.connect-to-cluster/","type":"doc"},{"title":"Customize installation defaults","content":"This topic introduces how to customize the default configurations when installing NebulaGraph Operator.\nCustomizable parameters\nWhen executing the helm install [NAME] [CHART] [flags] command to install a chart, you can specify the chart configuration. For more information, see Customizing the Chart Before Installing.\nYou can view the configurable options in the nebula-operator chart configuration file. Alternatively, you can view the configurable options through the command helm show values nebula-operator/nebula-operator, as shown below.\n[root@master ~]$ helm show values nebula-operator/nebula-operator \nimage:\n nebulaOperator:\n image: vesoft/nebula-operator:v1.7.3\n imagePullPolicy: Always\nimagePullSecrets: [ ]\nkubernetesClusterDomain: \"\"\ncontrollerManager:\n create: true\n replicas: 2\n env: [ ]\n resources:\n limits:\n cpu: 200m\n memory: 200Mi\n requests:\n cpu: 100m\n memory: 100Mi\nadmissionWebhook:\n create: false\n # The TCP port the Webhook server binds to. (default 9443)\n webhookBindPort: 9443\nscheduler:\n create: true\n schedulerName: nebula-scheduler\n replicas: 2\n env: [ ]\n resources:\n limits:\n cpu: 200m\n memory: 200Mi\n requests:\n cpu: 100m\n memory: 100Mi\n verbosity: 0\n plugins:\n enabled: [\"NodeZone\"]\n disabled: [] \n...\nPart of the above parameters are described as follows:\nParameter\nDefault value\nDescription\nimage.nebulaOperator.image\nvesoft/nebula-operator:v1.7.3\nThe image of NebulaGraph Operator, version of which is 1.7.3.\nimage.nebulaOperator.imagePullPolicy\nIfNotPresent\nThe image pull policy in Kubernetes.\nimagePullSecrets\nThe image pull secret in Kubernetes. For example imagePullSecrets[0].name=\"vesoft\".\nkubernetesClusterDomain\ncluster.local\nThe cluster domain.\ncontrollerManager.create\ntrue\nWhether to enable the controller-manager component.\ncontrollerManager.replicas\nThe number of controller-manager replicas.\nadmissionWebhook.create\nfalse\nWhether to enable Admission Webhook. This option is disabled. To enable it, set the value to true and you will need to install cert-manager. For details, see Enable admission control.\nshceduler.create\ntrue\nWhether to enable Scheduler.\nshceduler.schedulerName\nnebula-scheduler\nThe name of the scheduler customized by NebulaGraph Operator. It is used to evenly distribute Storage Pods across different zones which are only available in the Enterprise Edition.\nshceduler.replicas\nThe number of nebula-scheduler replicas.\nExample\nThe following example shows how to enable AdmissionWebhook when you install NebulaGraph Operator (AdmissionWebhook is disabled by default):\nhelm install nebula-operator nebula-operator/nebula-operator --namespace= --set admissionWebhook.create=true\nCheck whether the specified configuration of NebulaGraph Operator is installed successfully:\nhelm get values nebula-operator -n \nExample output:\nUSER-SUPPLIED VALUES:\nadmissionWebhook:\n create: true\nFor more information about helm install, see Helm Install.","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/3.operator-management/3.1.customize-installation/","type":"doc"},{"title":"Update NebulaGraph Operator","content":"This topic introduces how to update the configuration of NebulaGraph Operator.\nSteps\nUpdate the information of available charts locally from chart repositories.\nhelm repo update\nView the default values of NebulaGraph Operator.\nhelm show values nebula-operator/nebula-operator\nUpdate NebulaGraph Operator by passing configuration parameters via --set.\n--set：Overrides values using the command line. For more configurable items, see Customize installation defaults.\nFor example, to enable the AdmissionWebhook, run the following command:\nhelm upgrade nebula-operator nebula-operator/nebula-operator --namespace=nebula-operator-system --version=1.7.3 --set admissionWebhook.create=true\nFor more information, see Helm upgrade.\nCheck whether the configuration of NebulaGraph Operator is updated successfully.\nhelm get values nebula-operator -n nebula-operator-system\nExample output:\nUSER-SUPPLIED VALUES:\nadmissionWebhook:\n create: true","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/3.operator-management/3.2.update-operator/","type":"doc"},{"title":"Upgrade NebulaGraph Operator","content":"Steps\nView the current version of NebulaGraph Operator.\nhelm list --all-namespaces\nExample output:\nNAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION\nnebula-operator nebula-operator-system 3 2023-11-06 12:06:24.742397418 +0800 CST deployed nebula-operator-1.7.0 1.7.0\nUpdate the information of available charts locally from chart repositories.\nhelm repo update\nView the latest version of NebulaGraph Operator.\nhelm search repo nebula-operator/nebula-operator\nExample output:\nNAME CHART VERSION APP VERSION DESCRIPTION\nnebula-operator/nebula-operator 1.7.3 1.7.3 Nebula Operator Helm chart for Kubernetes\nUpgrade NebulaGraph Operator to version 1.7.3.\nhelm upgrade nebula-operator nebula-operator/nebula-operator --namespace= --version=1.7.3\nFor example:\nhelm upgrade nebula-operator nebula-operator/nebula-operator --namespace=nebula-operator-system --version=1.7.3\nOutput:\nRelease \"nebula-operator\" has been upgraded. Happy Helming!\nNAME: nebula-operator\nLAST DEPLOYED: Tue Apr 16 02:21:08 2022\nNAMESPACE: nebula-operator-system\nSTATUS: deployed\nREVISION: 3\nTEST SUITE: None\nNOTES:\nNebulaGraph Operator installed!\nPull the latest CRD configuration file.\nPull the NebulaGraph Operator chart package.\nhelm pull nebula-operator/nebula-operator --version=1.7.3\n--version: The NebulaGraph Operator version you want to upgrade to. If not specified, the latest version will be pulled.\nRun tar -zxvf to unpack the charts.\nFor example: To unpack v1.7.3 chart to the /tmp path, run the following command:\ntar -zxvf nebula-operator-1.7.3.tgz -C /tmp\n-C /tmp: If not specified, the chart files will be unpacked to the current directory.\nApply the latest CRD configuration file in the nebula-operator directory.\nkubectl apply -f crds/nebulaclusters.yaml\nOutput:\ncustomresourcedefinition.apiextensions.k8s.io/nebulaclusters.apps.nebula-graph.io configured","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/3.operator-management/3.3.upgrade-operator/","type":"doc"},{"title":"Uninstall NebulaGraph Operator","content":"This topic introduces how to uninstall NebulaGraph Operator.\nSteps\nUninstall the NebulaGraph Operator chart.\nhelm uninstall nebula-operator --namespace=\nView the information about the default-created CRD.\nkubectl get crd\nOutput:\nNAME CREATED AT\nnebulaautoscalers.autoscaling.nebula-graph.io 2023-11-01T04:16:51Z\nnebulaclusters.apps.nebula-graph.io 2023-10-12T07:55:32Z\nnebularestores.apps.nebula-graph.io 2023-02-04T23:01:00Z\nDelete CRD.\nkubectl delete crd nebulaclusters.apps.nebula-graph.io nebularestores.apps.nebula-graph.io nebulaautoscalers.autoscaling.nebula-graph.io","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/3.operator-management/3.4.unistall-operator/","type":"doc"},{"title":"Install a NebulaGraph cluster using NebulaGraph Operator","content":"Using NebulaGraph Operator to install NebulaGraph clusters enables automated cluster management with automatic error recovery. This topic covers two methods, kubectl apply and helm, for installing clusters using NebulaGraph Operator.\nPrerequisites\nInstall NebulaGraph Operator\nCreate a StorageClass\nUse kubectl apply\nCreate a namespace for storing NebulaGraph cluster-related resources. For example, create the nebula namespace.\nkubectl create namespace nebula\nCreate a YAML configuration file nebulacluster.yaml for the cluster. For example, create a cluster named nebula.\nExpand to view an example configuration for the nebula cluster\napiVersion: apps.nebula-graph.io/v1alpha1\nkind: NebulaCluster\nmetadata:\n name: nebula\n namespace: default\nspec:\n # Control the Pod scheduling strategy.\n topologySpreadConstraints:\n - topologyKey: \"kubernetes.io/hostname\"\n whenUnsatisfiable: \"ScheduleAnyway\"\n # Enable PV recycling.\n enablePVReclaim: false\n # Enable monitoring.\n exporter:\n image: vesoft/nebula-stats-exporter\n version: v3.3.0\n replicas: 1\n maxRequests: 20\n # Custom Agent image for cluster backup and restore, and log cleanup.\n agent:\n image: vesoft/nebula-agent\n version: latest\n resources:\n requests:\n cpu: \"100m\"\n memory: \"128Mi\"\n limits:\n cpu: \"200m\"\n memory: \"256Mi\" \n # Configure the image pull policy.\n imagePullPolicy: Always\n # Select the nodes for Pod scheduling.\n nodeSelector:\n nebula: cloud\n # Dependent controller name.\n reference:\n name: statefulsets.apps\n version: v1\n # Scheduler name.\n schedulerName: default-scheduler \n # Start NebulaGraph Console service for connecting to the Graph service.\n console:\n image: vesoft/nebula-console\n version: nightly\n username: \"demo\"\n password: \"test\" \n # Graph service configuration. \n graphd:\n # Used to check if the Graph service is running normally.\n # readinessProbe:\n # failureThreshold: 3\n # httpGet:\n # path: /status\n # port: 19669\n # scheme: HTTP\n # initialDelaySeconds: 40\n # periodSeconds: 10\n # successThreshold: 1\n # timeoutSeconds: 10\n # Container image for the Graph service.\n image: vesoft/nebula-graphd\n logVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n # Storage class name for storing Graph service logs.\n storageClassName: local-sc\n # Number of replicas for the Graph service Pod.\n replicas: 1\n # Resource configuration for the Graph service.\n resources:\n limits:\n cpu: \"1\"\n memory: 1Gi\n requests:\n cpu: 500m\n memory: 500Mi\n # Version of the Graph service.\n version: v3.6.0\n # Custom flags configuration for the Graph service.\n config: {}\n # Meta service configuration.\n metad:\n # readinessProbe:\n # failureThreshold: 3\n # httpGet:\n # path: /status\n # port: 19559\n # scheme: HTTP\n # initialDelaySeconds: 5\n # periodSeconds: 5\n # successThreshold: 1\n # timeoutSeconds: 5\n # Container image for the Meta service.\n image: vesoft/nebula-metad\n logVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: local-sc\n dataVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: local-sc\n replicas: 1\n resources:\n limits:\n cpu: \"1\"\n memory: 1Gi\n requests:\n cpu: 500m\n memory: 500Mi\n version: v3.6.0\n # Custom flags configuration for the Meta service.\n config: {} \n # Storage service configuration.\n storaged:\n # readinessProbe:\n # failureThreshold: 3\n # httpGet:\n # path: /status\n # port: 19779\n # scheme: HTTP\n # initialDelaySeconds: 40\n # periodSeconds: 10\n # successThreshold: 1\n # timeoutSeconds: 5\n # Container image for the Storage service.\n image: vesoft/nebula-graphd\n logVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: local-sc\n dataVolumeClaims:\n - resources:\n requests:\n storage: 2Gi\n storageClassName: local-sc\n replicas: 1\n resources:\n limits:\n cpu: \"1\"\n memory: 1Gi\n requests:\n cpu: 500m\n memory: 500Mi\n version: v3.6.0\n # Custom flags configuration for the Storage service.\n config: {} \nFor more detailed information about these parameters, see the Cluster configuration parameters section below.\nCreate the NebulaGraph cluster.\nkubectl create -f nebulacluster.yaml -n nebula\nOutput:\nnebulacluster.apps.nebula-graph.io/nebula created\nIf you don't specify the namespace using -n, it will default to the default namespace.\nCheck the status of the NebulaGraph cluster.\nkubectl get nebulaclusters nebula -n nebula\nOutput:\nNAME READY GRAPHD-DESIRED GRAPHD-READY METAD-DESIRED METAD-READY STORAGED-DESIRED STORAGED-READY AGE\nnebula True 1 1 1 1 1 1 86s\nUse helm\nAdd the NebulaGraph Operator Helm repository (if it's already added, run the next step directly).\nhelm repo add nebula-operator https://vesoft-inc.github.io/nebula-operator/charts\nUpdate the Helm repository to fetch the latest resources.\nhelm repo update nebula-operator\nSet environment variables for the configuration parameters required for installing the cluster.\nexport NEBULA_CLUSTER_NAME=nebula # Name of the NebulaGraph cluster.\nexport NEBULA_CLUSTER_NAMESPACE=nebula # Namespace for the NebulaGraph cluster.\nexport STORAGE_CLASS_NAME=local-sc # StorageClass for the NebulaGraph cluster.\nCreate a namespace for the NebulaGraph cluster if it is not created.\nkubectl create namespace \"${NEBULA_CLUSTER_NAMESPACE}\"\nCheck the customizable configuration parameters for the nebula-cluster Helm chart of the nebula-operator when creating the cluster.\nRun the following command to view all the configurable parameters.\nhelm show values nebula-operator/nebula-cluster\nVisit nebula-cluster/values.yaml to see all the configuration parameters for the NebulaGraph cluster. Click on Chart parameters to see the parameter descriptions and their default values.\nCreate the NebulaGraph cluster.\nYou can use the --set flag to customize the default values of the NebulaGraph cluster configuration. For example, --set nebula.storaged.replicas=3 sets the number of replicas for the Storage service to 3.\nhelm install \"${NEBULA_CLUSTER_NAME}\" nebula-operator/nebula-cluster \\ \n # Specify the version of the cluster chart. If not specified, it will install the latest version by default.\n # You can check all chart versions by running the command: helm search repo -l nebula-operator/nebula-cluster\n --version=1.7.3 \\\n # Specify the namespace for the NebulaGraph cluster.\n --namespace=\"${NEBULA_CLUSTER_NAMESPACE}\" \\\n # Customize the cluster name.\n --set nameOverride=\"${NEBULA_CLUSTER_NAME}\" \\\n --set nebula.storageClassName=\"${STORAGE_CLASS_NAME}\" \\\n # Specify the version for the NebulaGraph cluster.\n --set nebula.version=v3.6.0\nCheck the status of NebulaGraph cluster pods.\nkubectl -n \"${NEBULA_CLUSTER_NAMESPACE}\" get pod -l \"app.kubernetes.io/cluster=${NEBULA_CLUSTER_NAME}\"\nOutput:\nNAME READY STATUS RESTARTS AGE\nnebula-exporter-854c76989c-mp725 1/1 Running 0 14h\nnebula-graphd-0 1/1 Running 0 14h\nnebula-graphd-1 1/1 Running 0 14h\nnebula-metad-0 1/1 Running 0 14h\nnebula-metad-1 1/1 Running 0 14h\nnebula-metad-2 1/1 Running 0 14h\nnebula-storaged-0 1/1 Running 0 14h\nnebula-storaged-1 1/1 Running 0 14h\nnebula-storaged-2 1/1 Running 0 14h\nCluster configuration parameters\nThe table below lists the configurable parameters and their descriptions for creating a NebulaGraph cluster using a YAML file.\nParameter\nDefault Value\nDescription\nmetadata.name\nThe name of the created NebulaGraph cluster.\nspec.console\nLaunches a Console container for connecting to the Graph service. For configuration details, see nebula-console.\nspec.topologySpreadConstraints\nControls the scheduling strategy for Pods. For more details, see Topology Spread Constraints. When the value of topologyKey is kubernetes.io/zone, the value of whenUnsatisfiable must be set to DoNotSchedule, and the value of spec.schedulerName should be nebula-scheduler.\nspec.graphd.replicas\nThe number of replicas for the Graphd service.\nspec.graphd.image\nvesoft/nebula-graphd\nThe container image for the Graphd service.\nspec.graphd.version\nv3.6.0\nThe version of the Graphd service.\nspec.graphd.service\nConfiguration for accessing the Graphd service via a Service.\nspec.graphd.logVolumeClaim.storageClassName\nThe storage class name for the log volume claim of the Graphd service. When using sample configuration, replace it with the name of the pre-created storage class. See Storage Classes for creating a storage class.\nspec.metad.replicas\nThe number of replicas for the Metad service.\nspec.metad.image\nvesoft/nebula-metad\nThe container image for the Metad service.\nspec.metad.version\nv3.6.0\nThe version of the Metad service.\nspec.metad.dataVolumeClaim.storageClassName\nStorage configuration for the data disk of the Metad service. When using sample configuration, replace it with the name of the pre-created storage class. See Storage Classes for creating a storage class.\nspec.metad.logVolumeClaim.storageClassName\nStorage configuration for the log disk of the Metad service. When using sample configuration, replace it with the name of the pre-created storage class. See Storage Classes for creating a storage class.\nspec.storaged.replicas\nThe number of replicas for the Storaged service.\nspec.storaged.image\nvesoft/nebula-storaged\nThe container image for the Storaged service.\nspec.storaged.version\nv3.6.0\nThe version of the Storaged service.\nspec.storaged.dataVolumeClaims.resources.requests.storage\nThe storage size for the data disk of the Storaged service. You can specify multiple data disks. When specifying multiple data disks, the paths are like /usr/local/nebula/data1, /usr/local/nebula/data2, and so on.\nspec.storaged.dataVolumeClaims.storageClassName\nStorage configuration for the data disks of the Storaged service. When using sample configuration, replace it with the name of the pre-created storage class. See Storage Classes for creating a storage class.\nspec.storaged.logVolumeClaim.storageClassName\nStorage configuration for the log disk of the Storaged service. When using sample configuration, replace it with the name of the pre-created storage class. See Storage Classes for creating a storage class.\nspec..securityContext\n{}\nDefines the permission and access control for the cluster containers to control access and execution of container operations. For details, see SecurityContext.\nspec.agent\n{}\nConfiguration for the Agent service used for backup and recovery, and log cleaning functions. If you don't customize this configuration, the default configuration is used.\nspec.reference.name\n{}\nThe name of the controller it depends on.\nspec.schedulerName\ndefault-scheduler\nThe name of the scheduler.\nspec.imagePullPolicy\nAlways\nThe image pull policy for NebulaGraph images. For more details on pull policies, please see Image pull policy.\nspec.logRotate\n{}\nLog rotation configuration. For details, see Managing Cluster Logs.\nspec.enablePVReclaim\nfalse\nDefines whether to automatically delete PVCs after deleting the cluster to release data. For details, see Reclaim PV.","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.1.installation/4.1.1.cluster-install/","type":"doc"},{"title":"Upgrade NebulaGraph clusters created with NebulaGraph Operator","content":"This topic introduces how to upgrade a NebulaGraph cluster created with NebulaGraph Operator.\nLimits\nOnly for upgrading the NebulaGraph clusters created with NebulaGraph Operator.\nOnly support upgrading the NebulaGraph version from 3.5.0 to 3.6.0.\nFor upgrading NebulaGraph Enterprise Edition clusters, contact us.\nPrerequisites\nYou have created a NebulaGraph cluster. For details, see Create a NebulaGraph cluster.\nUpgrade a NebulaGraph cluster with kubectl\nThe following steps upgrade a NebulaGraph cluster from version 3.5.0 to v3.6.0.\nCheck the image version of the services in the cluster.\nkubectl get pods -l app.kubernetes.io/cluster=nebula -o jsonpath=\"{.items[*].spec.containers[*].image}\" |tr -s '[[:space:]]' '\\n' |sort |uniq -c\nOutput:\n 1 vesoft/nebula-graphd:3.5.0\n 1 vesoft/nebula-metad:3.5.0\n 3 vesoft/nebula-storaged:3.5.0 \nEdit the nebula cluster configuration to change the version value of the cluster services from 3.5.0 to v3.6.0.\nOpen the YAML file for the nebula cluster.\nkubectl edit nebulacluster nebula -n \nChange the value of version.\nAfter making these changes, the YAML file should look like this:\napiVersion: apps.nebula-graph.io/v1alpha1\nkind: NebulaCluster\nmetadata:\n name: nebula\nspec:\n graphd:\n version: v3.6.0 // Change the value from 3.5.0 to v3.6.0.\n ...\n metad:\n version: v3.6.0 // Change the value from 3.5.0 to v3.6.0.\n ...\n storaged:\n version: v3.6.0 // Change the value from 3.5.0 to v3.6.0.\n ...\nApply the configuration.\nAfter saving the YAML file and exiting, Kubernetes automatically updates the cluster's configuration and starts the cluster upgrade.\nAfter waiting for about 2 minutes, run the following command to see if the image versions of the services in the cluster have been changed to v3.6.0.\nkubectl get pods -l app.kubernetes.io/cluster=nebula -o jsonpath=\"{.items[*].spec.containers[*].image}\" |tr -s '[[:space:]]' '\\n' |sort |uniq -c\nOutput:\n 1 vesoft/nebula-graphd:v3.6.0\n 1 vesoft/nebula-metad:v3.6.0\n 3 vesoft/nebula-storaged:v3.6.0 \nUpgrade a NebulaGraph cluster with helm\nUpdate the information of available charts locally from chart repositories.\nhelm repo update\nSet environment variables to your desired values.\nexport NEBULA_CLUSTER_NAME=nebula # The desired NebulaGraph cluster name.\nexport NEBULA_CLUSTER_NAMESPACE=nebula # The desired namespace where your NebulaGraph cluster locates.\nUpgrade a NebulaGraph cluster.\nFor example, upgrade a cluster to v3.6.0.\nhelm upgrade \"${NEBULA_CLUSTER_NAME}\" nebula-operator/nebula-cluster \\\n --namespace=\"${NEBULA_CLUSTER_NAMESPACE}\" \\\n --set nameOverride=${NEBULA_CLUSTER_NAME} \\\n --set nebula.version=v3.6.0\nThe value of --set nebula.version specifies the version of the cluster you want to upgrade to.\nRun the following command to check the status and version of the upgraded cluster.\nCheck cluster status:\n$ kubectl -n \"${NEBULA_CLUSTER_NAMESPACE}\" get pod -l \"app.kubernetes.io/cluster=${NEBULA_CLUSTER_NAME}\"\nNAME READY STATUS RESTARTS AGE\nnebula-graphd-0 1/1 Running 0 2m\nnebula-graphd-1 1/1 Running 0 2m\nnebula-metad-0 1/1 Running 0 2m\nnebula-metad-1 1/1 Running 0 2m\nnebula-metad-2 1/1 Running 0 2m\nnebula-storaged-0 1/1 Running 0 2m\nnebula-storaged-1 1/1 Running 0 2m\nnebula-storaged-2 1/1 Running 0 2m\nCheck cluster version:\n$ kubectl get pods -l app.kubernetes.io/cluster=nebula -o jsonpath=\"{.items[*].spec.containers[*].image}\" |tr -s '[[:space:]]' '\\n' |sort |uniq -c\n 1 vesoft/nebula-graphd:v3.6.0\n 1 vesoft/nebula-metad:v3.6.0\n 3 vesoft/nebula-storaged:v3.6.0\nAccelerate the upgrade process\nThe upgrade process of a cluster is a rolling update process and can be time-consuming due to the state transition of the leader partition replicas in the Storage service. You can configure the enableForceUpdate field in the cluster instance's YAML file to skip the leader partition replica transfer operation, thereby accelerating the upgrade process. For more information, see Specify a rolling update strategy.\nTroubleshooting\nIf you encounter issues during the upgrade process, you can check the logs of the cluster service pods.\nkubectl logs -n \nAdditionally, you can inspect the cluster's status and events.\nkubectl describe nebulaclusters -n ","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.1.installation/4.1.2.cluster-upgrade/","type":"doc"},{"title":"Delete a NebulaGraph cluster","content":"This topic explains how to delete a NebulaGraph cluster created using NebulaGraph Operator.\nUsage limitations\nDeletion is only supported for NebulaGraph clusters created with the NebulaGraph Operator.\nDelete a NebulaGraph cluster using kubectl\nView all created clusters.\nkubectl get nc --all-namespaces\nExample output:\nNAMESPACE NAME READY GRAPHD-DESIRED GRAPHD-READY METAD-DESIRED METAD-READY STORAGED-DESIRED STORAGED-READY AGE\ndefault nebula True 2 2 3 3 3 3 38h\nnebula nebula2 True 1 1 1 1 1 1 2m7s\nDelete a cluster. For example, run the following command to delete a cluster named nebula2:\nkubectl delete nc nebula2 -n nebula\nExample output:\nnebulacluster.nebula-graph.io \"nebula2\" deleted\nConfirm the deletion.\nkubectl get nc nebula2 -n nebula\nExample output:\nNo resources found in nebula namespace.\nDelete a NebulaGraph cluster using helm\nView all Helm releases.\nhelm list --all-namespaces\nExample output:\nNAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION\nnebula default 1 2023-11-06 20:16:07.913136377 +0800 CST deployed nebula-cluster-1.7.1 1.7.1\nnebula-operator nebula-operator-system 3 2023-11-06 12:06:24.742397418 +0800 CST deployed nebula-operator-1.7.1 1.7.1\nView detailed information about a Helm release. For example, to view the cluster information for a Helm release named nebula:\nhelm get values nebula -n default\nExample output:\nUSER-SUPPLIED VALUES:\nimagePullSecrets:\n- name: secret_for_pull_image\nnameOverride: nebula # The cluster name\nnebula:\n graphd:\n image: reg.vesoft-inc.com/xx\n metad:\n image: reg.vesoft-inc.com/xx\n licenseManagerURL: xxx:9119\n storageClassName: local-sc\n storaged:\n image: reg.vesoft-inc.com/xx\n version: v1.7.3 # The cluster version\nUninstall a Helm release. For example, to uninstall a Helm release named nebula:\nhelm uninstall nebula -n default\nExample output:\nrelease \"nebula\" uninstalled\nOnce the Helm release is uninstalled, NebulaGraph Operator will automatically remove all K8s resources associated with that release.\nVerify that the cluster resources are removed.\nkubectl get nc nebula -n default\nExample output:\nNo resources found in default namespace.","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.1.installation/4.1.3.cluster-uninstall/","type":"doc"},{"title":"Customize the configuration of the NebulaGraph cluster","content":"The Meta, Storage, and Graph services each have their default configurations within the NebulaGraph cluster. NebulaGraph Operator allows for the customization of these cluster service configurations. This topic describes how to update the settings of the NebulaGraph cluster.\nPrerequisites\nA cluster is created using NebulaGraph Operator. For details, see Create a NebulaGraph Cluster.\nConfiguration method\nYou can update the configurations of cluster services by customizing parameters through spec..config. NebulaGraph Operator loads the configurations from config into the corresponding service's ConfigMap, which is then mounted into the service's configuration file directory (/usr/local/nebula/etc/) at the time of the service launch.\nThe structure of config is as follows:\nConfig map[string]string `json:\"config,omitempty\"`\nFor instance, when updating the Graph service's enable_authorize parameter settings, the spec.graphd.config parameter can be specified at the time of cluster creation, or during cluster runtime.\napiVersion: apps.nebula-graph.io/v1alpha1\nkind: NebulaCluster\nmetadata:\n name: nebula\n namespace: default\nspec:\n graphd:\n ...\n config: // Custom-defined parameters for the Graph service.\n \"enable_authorize\": \"true\" // Enable authorization. Default value is false.\n...\nIf you need to configure config for the Meta and Storage services, add corresponding configuration items to spec.metad.config and spec.storaged.config.\nConfigurable parameters\nFor more detailed information on the parameters that can be set under the config field, see the following:\n- Meta Service Configuration Parameters\n- Storage Service Configuration Parameters\n- Graph Service Configuration Parameters\nParameter updates & Pod restart rules\nConfiguration parameters for cluster services fall into two categories: those which require a service restart for any updates; and those which can be dynamically updated during service runtime. For the latter type, the updates will not be saved; subsequent to a service restart, configurations will revert to the state as shown in the configuration file.\nRegarding if the configuration parameters support dynamic updates during service runtime, please verify the information within the Whether supports runtime dynamic modifications column on each of the service configuration parameter detail pages linked above or see Dynamic runtime flags.\nDuring the update of cluster service configurations, keep the following points in mind:\nIf the updated parameters under config all allow for dynamic runtime updates, a service Pod restart will not be triggered and the configuration parameter updates will not be saved.\nIf the updated parameters under config include one or more that don’t allow for dynamic runtime updates, a service Pod restart will be triggered, but only updates to those parameters that don’t allow for dynamic updates will be saved.\nCustomize port configuration\nThe following example demonstrates how to customize the port configurations for the Meta, Storage, and Graph services.\nYou can add port and ws_http_port parameters to the config field in order to set custom ports. For detailed information regarding these two parameters, see the networking configuration sections at Meta Service Configuration Parameters, Storage Service Configuration Parameters, Graph Service Configuration Parameters.\nModify the cluster configuration file.\nOpen the cluster configuration file.\nkubectl edit nc nebula\nModify the configuration file as follows. \nAdd the config field to the graphd, metad, and storaged sections to customize the port configurations for the Graph, Meta, and Storage services, respectively.\napiVersion: apps.nebula-graph.io/v1alpha1\nkind: NebulaCluster\nmetadata:\n name: nebula\n namespace: default\nspec:\n graphd:\n config: // Custom port configuration for the Graph service.\n port: \"3669\"\n ws_http_port: \"8080\"\n resources:\n requests:\n cpu: \"200m\"\n memory: \"500Mi\"\n limits:\n cpu: \"1\"\n memory: \"1Gi\"\n replicas: 1\n image: vesoft/nebula-graphd\n version: v3.6.0\n metad: \n config: // Custom port configuration for the Meta service.\n ws_http_port: 8081\n resources:\n requests:\n cpu: \"300m\"\n memory: \"500Mi\"\n limits:\n cpu: \"1\"\n memory: \"1Gi\"\n replicas: 1\n image: vesoft/nebula-metad\n version: v3.6.0\n dataVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: local-path\n storaged: \n config: // Custom port configuration for the Storage service.\n ws_http_port: 8082\n resources:\n requests:\n cpu: \"300m\"\n memory: \"500Mi\"\n limits:\n cpu: \"1\"\n memory: \"1Gi\"\n replicas: 1\n image: vesoft/nebula-storaged\n version: v3.6.0\n dataVolumeClaims:\n - resources:\n requests:\n storage: 2Gi\n storageClassName: local-path\n enableAutoBalance: true\n reference:\n name: statefulsets.apps\n version: v1\n schedulerName: default-scheduler\n imagePullPolicy: IfNotPresent\n imagePullSecrets:\n - name: nebula-image\n enablePVReclaim: true\n topologySpreadConstraints:\n - topologyKey: kubernetes.io/hostname\n whenUnsatisfiable: \"ScheduleAnyway\"\nSave the changes.\nChanges will be saved automatically after saving the file.\nPress Esc to enter command mode.\nEnter :wq to save and exit.\nValidate that the configurations have taken effect.\nkubectl get svc\nExample output:\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\nnebula-graphd-headless ClusterIP None 3669/TCP,8080/TCP 10m\nnebula-graphd-svc ClusterIP 10.102.13.115 3669/TCP,8080/TCP 10m\nnebula-metad-headless ClusterIP None 9559/TCP,8081/TCP 11m\nnebula-storaged-headless ClusterIP None 9779/TCP,8082/TCP,9778/TCP 11m\nAs can be noticed, the Graph service's RPC daemon port is changed to 3669 (default 9669), the HTTP port to 8080 (default 19669); the Meta service's HTTP port is changed to 8081 (default 19559); the Storage service's HTTP port is changed to 8082 (default 19779).","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.2.configuration/","type":"doc"},{"title":"Dynamically expand persistent volumes","content":"In a Kubernetes environment, NebulaGraph's data is stored on Persistent Volumes (PVs). Dynamic volume expansion refers to increasing the capacity of a volume without stopping the service, enabling NebulaGraph to accommodate growing data. This topic explains how to dynamically expand the PV for NebulaGraph services in a Kubernetes environment.\nBackground\nIn Kubernetes, a StorageClass is a resource that defines a particular storage type. It describes a class of storage, including its provisioner, parameters, and other details. When creating a PersistentVolumeClaim (PVC) and specifying a StorageClass, Kubernetes automatically creates a corresponding PV. The principle of dynamic volume expansion is to edit the PVC and increase the volume's capacity. Kubernetes will then automatically expand the capacity of the PV associated with this PVC based on the specified storageClassName in the PVC. During this process, new PVs are not created; the size of the existing PV is changed. Only dynamic storage volumes, typically those associated with a storageClassName, support dynamic volume expansion. Additionally, the allowVolumeExpansion field in the StorageClass must be set to true. For more details, see the Kubernetes documentation on expanding Persistent Volume Claims.\nIn NebulaGraph Operator, you cannot directly edit PVC because Operator automatically creates PVC based on the configuration in the spec..dataVolumeClaim of the Nebula Graph cluster. Therefore, you need to modify the cluster's configuration to update the PVC and trigger dynamic online volume expansion for the PV.\nPrerequisites\nKubernetes version is equal to or greater than 1.18.\nA StorageClass has been created in the Kubernetes environment. For details, see Expanding Persistent Volumes Claims.\nEnsure the allowVolumeExpansion field in the StorageClass is set to true.\nMake sure that the provisioner configured in the StorageClass supports dynamic expansion.\nA NebulaGraph cluster is created in Kubernetes. For specific steps, see Create a NebulaGraph cluster.\nNebulaGraph cluster Pods are in running status.\nOnline volume expansion example\nIn the following example, we assume that the StorageClass is named ebs-sc and the NebulaGraph cluster is named nebula. We will demonstrate how to dynamically expand the PV for the Storage service.\nCheck the status of the Storage service Pod:\nkubectl get pod\nExample output:\nnebula-storaged-0 1/1 Running 0 43h\nCheck the PVC and PV information for the Storage service:\n# View PVC \nkubectl get pvc\nExample output:\nstoraged-data-nebula-storaged-0 Bound pvc-36ca3871-9265-460f-b812-7e73a718xxxx 5Gi RWO ebs-sc 43h\n# View PV and confirm that the capacity of the PV is 5Gi\nkubectl get pv\nExample output:\npvc-36ca3871-9265-460f-b812-xxx 5Gi RWO Delete Bound default/storaged-data-nebula-storaged-0 ebs-sc 43h\nAssuming all the above-mentioned prerequisites are met, use the following command to request an expansion of the PV for the Storage service to 10Gi:\nkubectl patch nc nebula --type='merge' --patch '{\"spec\": {\"storaged\": {\"dataVolumeClaims\":[{\"resources\": {\"requests\": {\"storage\": \"10Gi\"}}, \"storageClassName\": \"ebs-sc\"}]}}}'\nExample output:\nnebulacluster.apps.nebula-graph.io/nebula patched\nAfter waiting for about a minute, check the expanded PVC and PV information:\nkubectl get pvc\nExample output:\nstoraged-data-nebula-storaged-0 Bound pvc-36ca3871-9265-460f-b812-7e73a718xxxx 10Gi RWO ebs-sc 43h\nkubectl get pv\nExample output:\npvc-36ca3871-9265-460f-b812-xxx 10Gi RWO Delete Bound default/storaged-data-nebula-storaged-0 ebs-sc 43h\nAs you can see, both the PVC and PV capacity have been expanded to 10Gi.","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.4.storage-management/4.4.2.pv-expansion/","type":"doc"},{"title":"Reclaim PVs","content":"NebulaGraph Operator uses PVs (Persistent Volumes) and PVCs (Persistent Volume Claims) to store persistent data. If you accidentally deletes a NebulaGraph cluster, by default, PV and PVC objects and the relevant data will be retained to ensure data security.\nYou can also define the automatic deletion of PVCs to release data by setting the parameter spec.enablePVReclaim to true in the configuration file of the cluster instance. As for whether PV will be deleted automatically after PVC is deleted, you need to customize the PV reclaim policy. See reclaimPolicy in StorageClass and PV Reclaiming for details.\nPrerequisites\nA NebulaGraph cluster is created in Kubernetes. For specific steps, see Create a NebulaGraph cluster.\nSteps\nThe following example uses a cluster named nebula and the cluster's configuration file named nebula_cluster.yaml to show how to set enablePVReclaim:\nRun the following command to edit the nebula cluster's configuration file.\nkubectl edit nebulaclusters.apps.nebula-graph.io nebula\nAdd enablePVReclaim and set its value to true under spec.\napiVersion: apps.nebula-graph.io/v1alpha1\nkind: NebulaCluster\nmetadata:\n name: nebula\nspec:\n enablePVReclaim: true //Set its value to true.\n graphd:\n image: vesoft/nebula-graphd\n logVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: fast-disks\n replicas: 1\n resources:\n limits:\n cpu: \"1\"\n memory: 1Gi\n requests:\n cpu: 500m\n memory: 500Mi\n version: v3.6.0\n imagePullPolicy: IfNotPresent\n metad:\n dataVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: fast-disks\n image: vesoft/nebula-metad\n logVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: fast-disks\n replicas: 1\n resources:\n limits:\n cpu: \"1\"\n memory: 1Gi\n requests:\n cpu: 500m\n memory: 500Mi\n version: v3.6.0\n nodeSelector:\n nebula: cloud\n reference:\n name: statefulsets.apps\n version: v1\n schedulerName: default-scheduler\n storaged:\n dataVolumeClaims:\n - resources:\n requests:\n storage: 2Gi\n storageClassName: fast-disks\n - resources:\n requests:\n storage: 2Gi\n storageClassName: fast-disks\n image: vesoft/nebula-storaged\n logVolumeClaim:\n resources:\n requests:\n storage: 2Gi\n storageClassName: fast-disks\n replicas: 3\n resources:\n limits:\n cpu: \"1\"\n memory: 1Gi\n requests:\n cpu: 500m\n memory: 500Mi\n version: v3.6.0\n... \nRun kubectl apply -f nebula_cluster.yaml to push your configuration changes to the cluster.\nAfter setting enablePVReclaim to true, the PVCs of the cluster will be deleted automatically after the cluster is deleted. If you want to delete the PVs, you need to set the reclaim policy of the PVs to Delete. ","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.4.storage-management/4.4.3.configure-pv-reclaim/","type":"doc"},{"title":"Manage cluster logs","content":"Running logs of NebulaGraph cluster services (graphd, metad, storaged) are generated and stored in the /usr/local/nebula/logs directory of each service container by default.\nView logs\nTo view the running logs of a NebulaGraph cluster, you can use the kubectl logs command. \nFor example, to view the running logs of the Storage service:\n// View the name of the Storage service Pod, nebula-storaged-0.\n$ kubectl get pods -l app.kubernetes.io/component=storaged\nNAME READY STATUS RESTARTS AGE\nnebula-storaged-0 1/1 Running 0 45h\n...\n// Enter the container storaged of the Storage service.\n$ kubectl exec -it nebula-storaged-0 -c storaged -- /bin/bash\n// View the running logs of the Storage service.\n$ cd /usr/local/nebula/logs\nClean logs\nRunning logs generated by cluster services during runtime will occupy disk space. To avoid occupying too much disk space, the NebulaGraph Operator uses a sidecar container to periodically clean and archive logs.\nTo facilitate log collection and management, each NebulaGraph service deploys a sidecar container responsible for collecting logs generated by the service container and sending them to the specified log disk. The sidecar container automatically cleans and archives logs using the logrotate tool.\nIn the YAML configuration file of the cluster instance, set spec.logRotate to enable log rotation and set timestamp_in_logfile_name to false to disable the timestamp in the log file name to implement log rotation for the target service. The timestamp_in_logfile_name parameter is configured under the spec..config field. By default, the log rotation feature is turned off. Here is an example of enabling log rotation for all services:\n...\nspec:\n graphd:\n config:\n # Whether to include a timestamp in the log file name. \n # You must set this parameter to false to enable log rotation. \n # It is set to true by default.\n \"timestamp_in_logfile_name\": \"false\"\n metad:\n config:\n \"timestamp_in_logfile_name\": \"false\"\n storaged:\n config:\n \"timestamp_in_logfile_name\": \"false\"\n logRotate: # Log rotation configuration\n # The number of times a log file is rotated before being deleted.\n # The default value is 5, and 0 means the log file will not be rotated before being deleted.\n rotate: 5\n # The log file is rotated only if it grows larger than the specified size. The default value is 200M.\n size: \"200M\"\nCollect logs\nIf you don't want to mount additional log disks to back up log files, or if you want to collect logs and send them to a log center using services like fluent-bit, you can configure logs to be output to standard error. The Operator uses the glog tool to log to standard error output.\nIn the YAML configuration file of the cluster instance, you can configure logging to standard error output in the config and env fields of each service.\n...\nspec:\n graphd:\n config:\n # Whether to redirect standard error to a separate output file. The default value is false, which means it is not redirected.\n redirect_stdout: \"false\"\n # The severity level of log content: INFO, WARNING, ERROR, and FATAL. The corresponding values are 0, 1, 2, and 3.\n stderrthreshold: \"0\"\n env: \n - name: GLOG_logtostderr # Write log to standard error output instead of a separate file.\n value: \"1\" # 1 represents writing to standard error output, and 0 represents writing to a file.\n image: vesoft/nebula-graphd\n replicas: 1\n resources:\n requests:\n cpu: 500m\n memory: 500Mi\n service:\n externalTrafficPolicy: Local\n type: NodePort\n version: v3.6.0\n metad:\n config:\n redirect_stdout: \"false\"\n stderrthreshold: \"0\"\n dataVolumeClaim:\n resources:\n requests:\n storage: 1Gi\n storageClassName: ebs-sc\n env:\n - name: GLOG_logtostderr\n value: \"1\"\n image: vesoft/nebula-metad\n ...","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.5.logging/","type":"doc"},{"title":"Enable admission control","content":"Kubernetes Admission Control is a security mechanism running as a webhook at runtime. It intercepts and modifies requests to ensure the cluster's security. Admission webhooks involve two main operations: validation and mutation. NebulaGraph Operator supports only validation operations and provides some default admission control rules. This topic describes NebulaGraph Operator's default admission control rules and how to enable admission control.\nPrerequisites\nA NebulaGraph cluster is created with NebulaGrpah Operator. For detailed steps, see Create a NebulaGraph cluster.\nAdmission control rules\nKubernetes admission control allows you to insert custom logic or policies before Kubernetes API Server processes requests. This mechanism can be used to implement various security policies, such as restricting a Pod's resource consumption or limiting its access permissions. NebulaGraph Operator supports validation operations, which means it validates and intercepts requests without making changes. NebulaGraph Operator's default admission validation control rules include:\nEnsuring the minimum number of replicas in high availability mode:\nFor Graph service: At least 2 replicas are required.\nFor Meta service: At least 3 replicas are required.\nFor Storage service: At least 3 replicas are required.\nPreventing additional PVs from being added to Storage service via dataVolumeClaims.\nDisallowing shrinking the capacity of all service's PVCs, but allowing expansion.\nForbidding any secondary operation during Storage service scale-in/scale-out.\nTLS certificates for admission webhooks\nTo ensure secure communication and data integrity between the K8s API server and the admission webhook, this communication is done over HTTPS by default. This means that TLS certificates are required for the admission webhook. cert-manager is a Kubernetes certificate management controller that automates the issuance and renewal of certificates. NebulaGraph Operator uses cert-manager to manage certificates.\nOnce cert-manager is installed and admission control is enabled, NebulaGraph Operator will automatically create an Issuer for issuing the necessary certificate for the admission webhook, and a Certificate for storing the issued certificate. The issued certificate is stored in the nebula-operator-webhook-secret Secret.\nSteps of enabling admission control\nInstall cert-manager.\nkubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.1/cert-manager.yaml\nIt is suggested to deploy the latest version of cert-manager. For details, see the official cert-manager documentation.\nModify the NebulaGraph Operator configuration file to enable admission control. Admission control is disabled by default and needs to be enabled manually.\n# Check the current configuration\nhelm show values nebula-operator/nebula-operator\n# Modify the configuration by setting `enableAdmissionWebhook` to `true`.\nhelm upgrade nebula-operator nebula-operator/nebula-operator --set enableAdmissionWebhook=true\nView the certificate Secret for the admission webhook.\nkubectl get secret nebula-operator-webhook-secret -o yaml\nIf the output includes certificate contents, it means that the admission webhook's certificate has been successfully created.\nVerify the control rules.\nVerify the minimum number of replicas in high availability mode.\n# Annotate the cluster to enable high availability mode.\n$ kubectl annotate nc nebula nebula-graph.io/ha-mode=true\n# Verify the minimum number of the Graph service's replicas.\n$ kubectl patch nc nebula --type='merge' --patch '{\"spec\": {\"graphd\": {\"replicas\":1}}}'\nError from server: admission webhook \"nebulaclustervalidating.nebula-graph.io\" denied the request: spec.graphd.replicas: Invalid value: 1: should be at least 2 in HA mode\nVerify preventing additional PVs from being added to Storage service.\n$ kubectl patch nc nebula --type='merge' --patch '{\"spec\": {\"storaged\": {\"dataVolumeClaims\":[{\"resources\": {\"requests\": {\"storage\": \"2Gi\"}}, \"storageClassName\": \"local-path\"},{\"resources\": {\"requests\": {\"storage\": \"3Gi\"}}, \"storageClassName\": \"fask-disks\"}]}}}'\nError from server: admission webhook \"nebulaclustervalidating.nebula-graph.io\" deniedthe request: spec.storaged.dataVolumeClaims: Forbidden: storaged dataVolumeClaims is immutable\nVerify disallowing shrinking Storage service's PVC capacity.\n$ kubectl patch nc nebula --type='merge' --patch '{\"spec\": {\"storaged\": {\"dataVolumeClaims\":[{\"resources\": {\"requests\": {\"storage\": \"1Gi\"}}, \"storageClassName\": \"fast-disks\"}]}}}'\nError from server: admission webhook \"nebulaclustervalidating.nebula-graph.io\" denied the request: spec.storaged.dataVolumeClaims: Invalid value: resource.Quantity{i:resource.int64Amount{value:1073741824, scale:0}, d:resource.infDecAmount{Dec:(*inf.Dec)(nil)}, s:\"1Gi\", Format:\"BinarySI\"}: data volume size can only be increased\nVerify disallowing any secondary operation during Storage service scale-in.\n$ kubectl patch nc nebula --type='merge' --patch '{\"spec\": {\"storaged\": {\"replicas\": 5}}}'\nnebulacluster.apps.nebula-graph.io/nebula patched\n$ kubectl patch nc nebula --type='merge' --patch '{\"spec\": {\"storaged\": {\"replicas\": 3}}}'\nError from server: admission webhook \"nebulaclustervalidating.nebula-graph.io\" denied the request: [spec.storaged: Forbidden: field is immutable while in ScaleOut phase, spec.storaged.replicas: Invalid value: 3: field is immutable while not in Running phase]","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.7.security/4.7.2.enable-admission-control/","type":"doc"},{"title":"Self-healing","content":"NebulaGraph Operator calls the interface provided by NebulaGraph clusters to dynamically sense cluster service status. Once an exception is detected (for example, a component in a NebulaGraph cluster stops running), NebulaGraph Operator automatically performs fault tolerance. This topic shows how Nebular Operator performs self-healing by simulating cluster failure of deleting one Storage service Pod in a NebulaGraph cluster.\nPrerequisites\nInstall NebulaGraph Operator\nSteps\nCreate a NebulaGraph cluster. For more information, see Create a NebulaGraph clusters.\nDelete the Pod named -storaged-2 after all pods are in the Running status.\nkubectl delete pod -storaged-2 --now\n is the name of your NebulaGraph cluster.\nNebulaGraph Operator automates the creation of the Pod named -storaged-2 to perform self-healing.\nRun the kubectl get pods command to check the status of the Pod -storaged-2.\n...\nnebula-cluster-storaged-1 1/1 Running 0 5d23h\nnebula-cluster-storaged-2 0/1 ContainerCreating 0 1s\n...\n...\nnebula-cluster-storaged-1 1/1 Running 0 5d23h\nnebula-cluster-storaged-2 1/1 Running 0 4m2s\n...\nWhen the status of -storaged-2 is changed from ContainerCreating to Running, the self-healing is performed successfully.","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.8.ha-and-balancing/4.8.1.self-healing/","type":"doc"},{"title":"Optimize leader transfer in rolling updates","content":"NebulaGraph clusters use a distributed architecture to divide data into multiple logical partitions, which are typically evenly distributed across different nodes. In distributed systems, there are usually multiple replicas of the same data. To ensure the consistency of data across multiple replicas, NebulaGraph clusters use the Raft protocol to synchronize multiple partition replicas. In the Raft protocol, each partition elects a leader replica, which is responsible for handling write requests, while follower replicas handle read requests.\nWhen a NebulaGraph cluster created by NebulaGraph Operator performs a rolling update, a storage node temporarily stops providing services for the update. For an overview of rolling updates, see Performing a Rolling Update. If the node hosting the leader replica stops providing services, it will result in the unavailability of read and write operations for that partition. To avoid this situation, by default, Operator migrates the leader replicas to other unaffected nodes during the rolling update process of a NebulaGraph cluster. This way, when a storage node is being updated, the leader replicas on other nodes can continue processing client requests, ensuring the read and write availability of the cluster.\nThe process of migrating all leader replicas from one storage node to the other nodes may take a long time. To better control the rolling update duration, Operator provides a field called enableForceUpdate. When it is confirmed that there is no external access traffic, you can set this field to true. This way, the leader replicas will not be migrated to other nodes, thereby speeding up the rolling update process.\nRolling update trigger conditions\nOperator triggers a rolling update of the NebulaGraph cluster under the following circumstances:\nThe version of the NebulaGraph cluster changes.\nThe configuration of the NebulaGraph cluster changes.\nNebulaGraph cluster services are restarted.\nSpecify a rolling update strategy\nIn the YAML file for creating a cluster instance, add the spec.storaged.enableForceUpdate field and set it to true or false to control the rolling update speed.\nWhen enableForceUpdate is set to true, it means that the partition leader replicas will not be migrated, thus speeding up the rolling update process. Conversely, when set to false, it means that the leader replicas will be migrated to other nodes to ensure the read and write availability of the cluster. The default value is false.\nConfiguration example:\n...\nspec:\n...\n storaged:\n # When set to true,\n # it means that the partition leader replicas will not be migrated,\n # but the cluster pods will be rebuilt directly.\n enableForceUpdate: true \n ...","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/4.cluster-administration/4.9.advanced/4.9.1.rolling-update-strategy/","type":"doc"},{"title":"FAQ","content":"Does NebulaGraph Operator support the v1.x version of NebulaGraph?\nNo, because the v1.x version of NebulaGraph does not support DNS, and NebulaGraph Operator requires the use of DNS.\nIs cluster stability guaranteed if using local storage?\nThere is no guarantee. Using local storage means that the Pod is bound to a specific node, and NebulaGraph Operator does not currently support failover in the event of a failure of the bound node.\nHow to ensure the stability of a cluster when scaling the cluster?\nIt is suggested to back up data in advance so that you can roll back data in case of failure.\nIs the replica in the Operator docs the same as the replica in the NebulaGraph core docs?\nThey are different concepts. A replica in the Operator docs indicates a pod replica in K8s, while a replica in the core docs is a replica of a NebulaGraph storage partition.\nHow to view the logs of each service in the NebulaGraph cluster?\nTo obtain the logs of each cluster service, you need to access the container and view the log files that are stored inside.\nSteps to view the logs of each service in the NebulaGraph cluster:\n# To view the name of the pod where the container you want to access is located. \n# Replace with the name of the cluster.\nkubectl get pods -l app.kubernetes.io/cluster=\n# To access the container within the pod, such as the nebula-graphd-0 container.\nkubectl exec -it nebula-graphd-0 -- /bin/bash\n# To go to /usr/local/nebula/logs directory to view the logs.\ncd /usr/local/nebula/logs\nHow to resolve the host not found:nebula--0.nebula.-headless.default.svc.cluster.local error?\nThis error is generally caused by a DNS resolution failure, and you need to check whether the cluster domain has been modified. If the cluster domain has been modified, you need to modify the kubernetesClusterDomain field in the NebulaGraph Operator configuration file accordingly. The steps for modifying the Operator configuration file are as follows:\nView the Operator configuration file.\n[abby@master ~]$ helm show values nebula-operator/nebula-operator \nimage:\n nebulaOperator:\n image: vesoft/nebula-operator:v1.7.3\n imagePullPolicy: Always\n kubeRBACProxy:\n image: bitnami/kube-rbac-proxy:0.14.2\n imagePullPolicy: Always\n kubeScheduler:\n image: registry.k8s.io/kube-scheduler:v1.24.11\n imagePullPolicy: Always\nimagePullSecrets: []\nkubernetesClusterDomain: \"\" # The cluster domain name, and the default is cluster.local.\nModify the value of the kubernetesClusterDomain field to the updated cluster domain name.\nhelm upgrade nebula-operator nebula-operator/nebula-operator --namespace= --version=1.7.3 --set kubernetesClusterDomain=\n is the namespace where Operator is located and is the updated domain name.","url":"https://docs.nebula-graph.io/3.6.0/k8s-operator/5.FAQ/","type":"doc"},{"title":"NebulaGraph Algorithm","content":"NebulaGraph Algorithm (Algorithm) is a Spark application based on GraphX. It uses a complete algorithm tool to perform graph computing on the data in the NebulaGraph database by submitting a Spark task. You can also programmatically use the algorithm under the lib repository to perform graph computing on DataFrame.\nVersion compatibility\nThe correspondence between the NebulaGraph Algorithm release and the NebulaGraph core release is as follows.\nNebulaGraph\nNebulaGraph Algorithm\nnightly\n3.0-SNAPSHOT\n3.0.0 ~ 3.4.x\n3.x.0\n2.6.x\n2.6.x\n2.5.0、2.5.1\n2.5.0\n2.0.0、2.0.1\n2.1.0\nPrerequisites\nBefore using the NebulaGraph Algorithm, users need to confirm the following information:\nThe NebulaGraph services have been deployed and started. For details, see NebulaGraph Installation.\nThe Spark version is 2.4.x.\nThe Scala version is 2.11.\n(Optional) If users need to clone, compile, and package the latest Algorithm in Github, install Maven.\nLimitations\nGraph computing outputs vertex datasets, and the algorithm results are stored in DataFrames as the properties of vertices. You can do further operations such as statistics and filtering according to your business requirements.\n!!!\nBefore Algorithm v3.1.0, when submitting the algorithm package directly, the data of the vertex ID must be an integer. That is, the vertex ID can be INT or String, but the data itself is an integer.\nSupported algorithms\nThe graph computing algorithms supported by NebulaGraph Algorithm are as follows.\nAlgorithm\nDescription\nScenario\nProperties name\nProperties type\nPageRank\nThe rank of pages\nWeb page ranking, key node mining\npagerank\ndouble/string\nLouvain\nLouvain\nCommunity mining, hierarchical clustering\nlouvain\nint/string\nKCore\nK core\nCommunity discovery, financial risk control\nkcore\nint/string\nLabelPropagation\nLabel propagation\nInformation spreading, advertising, and community discovery\nlpa\nint/string\nHanp\nLabel propagation advanced\nCommunity discovery, recommendation system\nhanp\nint/string\nConnectedComponent\nWeakly connected component\nCommunity discovery, island discovery\ncc\nint/string\nStronglyConnectedComponent\nStrongly connected component\nCommunity discovery\nscc\nint/string\nShortestPath\nThe shortest path\nPath planning, network planning\nshortestpath\nstring\nTriangleCount\nTriangle counting\nNetwork structure analysis\ntrianglecount\nint/string\nGraphTriangleCount\nGraph triangle counting\nNetwork structure and tightness analysis\ncount\nint\nBetweennessCentrality\nIntermediate centrality\nKey node mining, node influence computing\nbetweenness\ndouble/string\nClosenessCentrality\nCloseness centrality\nKey node mining, node influence computing\ncloseness\ndouble/string\nDegreeStatic\nDegree of statistical\nGraph structure analysis\ndegree,inDegree,outDegree\nint/string\nClusteringCoefficient\nAggregation coefficient\nRecommendation system, telecom fraud analysis\nclustercoefficient\ndouble/string\nJaccard\nJaccard similarity\nSimilarity computing, recommendation system\njaccard\nstring\nBFS\nBreadth-First Search\nSequence traversal, shortest path planning\nbfs\nstring\nDFS\nDepth-First Search\nSequence traversal, shortest path planning\ndfs\nstring\nNode2Vec\nGraph classification\nnode2vec\nstring\nImplementation methods\nNebulaGraph Algorithm implements the graph calculating as follows:\nRead the graph data of DataFrame from the NebulaGraph database using the NebulaGraph Spark Connector.\nTransform the graph data of DataFrame to the GraphX graph.\nUse graph algorithms provided by GraphX (such as PageRank) or self-implemented algorithms (such as Louvain).\nFor detailed implementation methods, see Scala file.\nGet NebulaGraph Algorithm\nCompile and package\nClone the repository nebula-algorithm.\n$ git clone -b v3.0.0 https://github.com/vesoft-inc/nebula-algorithm.git\nEnter the directory nebula-algorithm.\n$ cd nebula-algorithm\nCompile and package.\n$ mvn clean package -Dgpg.skip -Dmaven.javadoc.skip=true -Dmaven.test.skip=true\nAfter the compilation, a similar file nebula-algorithm-3.x.x.jar is generated in the directory nebula-algorithm/target.\nDownload maven from the remote repository\nDownload address\nHow to use\nUse algorithm interface (recommended)\nThe lib repository provides 10 common graph algorithms.\nAdd dependencies to the file pom.xml.\n\n com.vesoft\n nebula-algorithm\n 3.0.0\n\nUse the algorithm (take PageRank as an example) by filling in parameters. For more examples, see example.\nval prConfig = new PRConfig(5, 1.0)\nval prResult = PageRankAlgo.apply(spark, data, prConfig, false)\nIf your vertex IDs are Strings, see Pagerank Example for how to encoding and decoding them.\nSubmit the algorithm package directly\nSet the Configuration file.\n # Configurations related to Spark\n spark: {\n app: {\n name: LPA\n # The number of partitions of Spark\n partitionNum:100\n master:local\n data: {\n # Data source. Optional values are nebula, csv, and json.\n source: csv\n # Data sink. The algorithm result will be written into this sink. Optional values are nebula, csv, and text.\n sink: nebula\n # Whether the algorithm has a weight.\n hasWeight: false\n # Configurations related to NebulaGraph\n nebula: {\n # Data source. When NebulaGraph is the data source of the graph computing, the configuration of `nebula.read` is valid.\n read: {\n # The IP addresses and ports of all Meta services. Multiple addresses are separated by commas (,). Example: \"ip1:port1,ip2:port2\".\n # To deploy NebulaGraph by using Docker Compose, fill in the port with which Docker Compose maps to the outside.\n # Check the status with `docker-compose ps`.\n metaAddress: \"192.168.*.10:9559\"\n # The name of the graph space in NebulaGraph.\n space: basketballplayer\n # Edge types in NebulaGraph. When there are multiple labels, the data of multiple edges will be merged.\n labels: [\"serve\"]\n # The property name of each edge type in NebulaGraph. This property will be used as the weight column of the algorithm. Make sure that it corresponds to the edge type.\n weightCols: [\"start_year\"]\n # Data sink. When the graph computing result sinks into NebulaGraph, the configuration of `nebula.write` is valid.\n write:{\n # The IP addresses and ports of all Graph services. Multiple addresses are separated by commas (,). Example: \"ip1:port1,ip2:port2\".\n # To deploy by using Docker Compose, fill in the port with which Docker Compose maps to the outside.\n # Check the status with `docker-compose ps`.\n graphAddress: \"192.168.*.11:9669\"\n # The IP addresses and ports of all Meta services. Multiple addresses are separated by commas (,). Example: \"ip1:port1,ip2:port2\".\n # To deploy NebulaGraph by using Docker Compose, fill in the port with which Docker Compose maps to the outside.\n # Check the staus with `docker-compose ps`.\n metaAddress: \"192.168.*.12:9559\"\n user:root\n pswd:nebula\n # Before submitting the graph computing task, create the graph space and tag.\n # The name of the graph space in NebulaGraph.\n space:nb\n # The name of the tag in NebulaGraph. The graph computing result will be written into this tag. The property name of this tag is as follows.\n # PageRank: pagerank\n # Louvain: louvain\n # ConnectedComponent: cc\n # StronglyConnectedComponent: scc\n # LabelPropagation: lpa\n # ShortestPath: shortestpath\n # DegreeStatic: degree,inDegree,outDegree\n # KCore: kcore\n # TriangleCount: tranglecpunt\n # BetweennessCentrality: betweennedss\n tag:pagerank\n local: {\n # Data source. When the data source is csv or json, the configuration of `local.read` is valid.\n read:{\n filePath: \"hdfs://127.0.0.1:9000/edge/work_for.csv\"\n # If the CSV file has a header or it is a json file, use the header. If not, use [_c0, _c1, _c2, ..., _cn] instead.\n # The header of the source VID column.\n srcId:\"_c0\"\n # The header of the destination VID column.\n dstId:\"_c1\"\n # The header of the weight column.\n weight: \"_c2\"\n # Whether the csv file has a header.\n header: false\n # The delimiter in the csv file.\n delimiter:\",\"\n # Data sink. When the graph computing result sinks to the csv or text file, the configuration of `local.write` is valid.\n write:{\n resultPath:/tmp/\n algorithm: {\n # The algorithm to execute. Optional values are as follow: \n # pagerank, louvain, connectedcomponent, labelpropagation, shortestpaths, \n # degreestatic, kcore, stronglyconnectedcomponent, trianglecount ,\n # betweenness, graphtriangleCount.\n executeAlgo: pagerank\n # PageRank\n pagerank: {\n maxIter: 10\n resetProb: 0.15 \n encodeId:false # Configure true if the VID is of string type.\n # Louvain\n louvain: {\n maxIter: 20\n internalIter: 10\n tol: 0.5\n encodeId:false # Configure true if the VID is of string type.\n # ...\nSubmit the graph computing task.\n${SPARK_HOME}/bin/spark-submit --master --class com.vesoft.nebula.algorithm.Main -p \nExample:\n${SPARK_HOME}/bin/spark-submit --master \"local\" --class com.vesoft.nebula.algorithm.Main /root/nebula-algorithm/target/nebula-algorithm-3.0-SNAPSHOT.jar -p /root/nebula-algorithm/src/main/resources/application.conf","url":"https://docs.nebula-graph.io/3.6.0/graph-computing/nebula-algorithm/","type":"doc"},{"title":"NebulaGraph Bench","content":"NebulaGraph Bench is a performance test tool for NebulaGraph using the LDBC data set.\nScenario\nGenerate test data and import NebulaGraph.\nPerformance testing in the NebulaGraph cluster.\nRelease note\nRelease\nTest process\nFor detailed usage instructions, see NebulaGraph Bench.","url":"https://docs.nebula-graph.io/3.6.0/nebula-bench/","type":"doc"},{"title":"FAQ","content":"This topic lists the frequently asked questions for using NebulaGraph 3.6.0. You can use the search box in the help center or the search function of the browser to match the questions you are looking for.\nIf the solutions described in this topic cannot solve your problems, ask for help on the NebulaGraph forum or submit an issue on GitHub issue.\nAbout manual updates\n\"Why is the behavior in the manual not consistent with the system?\"\nNebulaGraph is still under development. Its behavior changes from time to time. Users can submit an issue to inform the team if the manual and the system are not consistent.\nAbout legacy version compatibility\nAbout execution errors\n\"How to resolve the error -1005:GraphMemoryExceeded: (-2600)?\"\nThis error is issued by the Memory Tracker when it observes that memory usage has exceeded a set threshold. This mechanism can help avoid service processes from being terminated by the system's OOM (Out of Memory) killer. Steps to resolve:\nCheck memory usage: First, you need to check the memory usage during the execution of the command. If the memory usage is indeed high, then this error might be expected.\nCheck the configuration of the Memory Tracker: If the memory usage is not high, check the relevant configurations of the Memory Tracker. These include memory_tracker_untracked_reserved_memory_mb (untracked reserved memory in MB), memory_tracker_limit_ratio (memory limit ratio), and memory_purge_enabled (whether memory purge is enabled). For the configuration of the Memory Tracker, see memory tracker configuration.\nOptimize configurations: Adjust these configurations according to the actual situation. For example, if the available memory limit is too low, you can increase the value of memory_tracker_limit_ratio.\n\"How to resolve the error SemanticError: Missing yield clause.?\"\nStarting with NebulaGraph 3.0.0, the statements LOOKUP, GO, and FETCH must output results with the YIELD clause. For more information, see YIELD.\n\"How to resolve the error Host not enough!?\"\nFrom NebulaGraph version 3.0.0, the Storage services added in the configuration files CANNOT be read or written directly. The configuration files only register the Storage services into the Meta services. You must run the ADD HOSTS command to read and write data on Storage servers. For more information, see Manage Storage hosts.\n\"How to resolve the error To get the property of the vertex in 'v.age', should use the format 'var.tag.prop'?\"\nFrom NebulaGraph version 3.0.0, patterns support matching multiple tags at the same time, so you need to specify a tag name when querying properties. The original statement RETURN variable_name.property_name is changed to RETURN variable_name..property_name.\n\"How to resolve the error Storage Error E_RPC_FAILURE?\"\nThe reason for this error is usually that the storaged process returns too many data back to the graphd process. Possible solutions are as follows:\nModify configuration files: Modify the value of --storage_client_timeout_ms in the nebula-graphd.conf file to extend the connection timeout of the Storage client. This configuration is measured in milliseconds (ms). For example, set --storage_client_timeout_ms=60000. If this parameter is not specified in the nebula-graphd.conf file, specify it manually. Tip: Add --local_config=true at the beginning of the configuration file and restart the service.\nOptimize the query statement: Reduce queries that scan the entire database. No matter whether LIMIT is used to limit the number of returned results, use the GO statement to rewrite the MATCH statement (the former is optimized, while the latter is not).\nCheck whether the Storaged process has OOM. (dmesg |grep nebula).\nUse better SSD or memory for the Storage Server.\nRetry.\n\"How to resolve the error The leader has changed. Try again later?\"\nIt is a known issue. Just retry 1 to N times, where N is the partition number. The reason is that the meta client needs some heartbeats to update or errors to trigger the new leader information.\nIf this error occurs when logging in to NebulaGraph, you can consider using df -h to view the disk space and check whether the local disk is full.\nUnable to download SNAPSHOT packages when compiling Exchange, Connectors, or Algorithm\nProblem description: The system reports Could not find artifact com.vesoft:client:jar:xxx-SNAPSHOT when compiling.\nCause: There is no local Maven repository for storing or downloading SNAPSHOT packages. The default central repository in Maven only stores official releases, not development versions (SNAPSHOTs).\nSolution: Add the following configuration in the profiles scope of Maven's setting.xml file:\n \n \n true\n \n \n \n snapshots\n https://oss.sonatype.org/content/repositories/snapshots/\n \n true\n \n \n \n \n\"How to resolve [ERROR (-1004)]: SyntaxError: syntax error near?\"\nIn most cases, a query statement requires a YIELD or a RETURN. Check your query statement to see if YIELD or RETURN is provided.\n\"How to resolve the error can’t solve the start vids from the sentence?\"\nThe graphd process requires start vids to begin a graph traversal. The start vids can be specified by the user. For example:\n> GO FROM ${vids} ...\n> MATCH (src) WHERE id(src) == ${vids}\n# The \"start vids\" are explicitly given by ${vids}.\nIt can also be found from a property index. For example:\n# CREATE TAG INDEX IF NOT EXISTS i_player ON player(name(20));\n# REBUILD TAG INDEX i_player;\n> LOOKUP ON player WHERE player.name == \"abc\" | ... YIELD ...\n> MATCH (src) WHERE src.name == \"abc\" ...\n# The \"start vids\" are found from the property index \"name\".\nOtherwise, an error like can’t solve the start vids from the sentence will be returned.\n\"How to resolve the error Wrong vertex id type: 1001?\"\nCheck whether the VID is INT64 or FIXED_STRING(N) set by create space. For more information, see create space.\n\"How to resolve the error The VID must be a 64-bit integer or a string fitting space vertex id length limit.?\"\nCheck whether the length of the VID exceeds the limitation. For more information, see create space.\n\"How to resolve the error edge conflict or vertex conflict?\"\nNebulaGraph may return such errors when the Storage service receives multiple requests to insert or update the same vertex or edge within milliseconds. Try the failed requests again later.\n\"How to resolve the error RPC failure in MetaClient: Connection refused?\"\nThe reason for this error is usually that the metad service status is unusual, or the network of the machine where the metad and graphd services are located is disconnected. Possible solutions are as follows:\nCheck the metad service status on the server where the metad is located. If the service status is unusual, restart the metad service.\nUse telnet meta-ip:port to check the network status under the server that returns an error.\nCheck the port information in the configuration file. If the port is different from the one used when connecting, use the port in the configuration file or modify the configuration.\n\"How to resolve the error StorageClientBase.inl:214] Request to \"x.x.x.x\":9779 failed: N6apache6thrift9transport19TTransportExceptionE: Timed Out in nebula-graph.INFO?\"\nThe reason for this error may be that the amount of data to be queried is too large, and the storaged process has timed out. Possible solutions are as follows:\nWhen importing data, set Compaction manually to make read faster.\nExtend the RPC connection timeout of the Graph service and the Storage service. Modify the value of --storage_client_timeout_ms in the nebula-graphd.conf file. This configuration is measured in milliseconds (ms). The default value is 60000ms.\n\"How to resolve the error MetaClient.cpp:65] Heartbeat failed, status:Wrong cluster! in nebula-storaged.INFO, or HBProcessor.cpp:54] Reject wrong cluster host \"x.x.x.x\":9771! in nebula-metad.INFO?\"\nThe reason for this error may be that the user has modified the IP or the port information of the metad process, or the storage service has joined other clusters before. Possible solutions are as follows:\nDelete the cluster.id file in the installation directory where the storage machine is deployed (the default installation directory is /usr/local/nebula), and restart the storaged service.\n\"How to resolve the error Storage Error: More than one request trying to add/update/delete one edge/vertex at he same time.?\"\nThe reason for this error is that the current NebulaGraph version does not support concurrent requests to the same vertex or edge at the same time. To solve this error, re-execute your commands.\nAbout design and functions\n\"How is the time spent value at the end of each return message calculated?\"\nTake the returned message of SHOW SPACES as an example:\n SHOW SPACES;\nGot 1 rows (time spent 1235/1934 us)\nThe first number 1235 shows the time spent by the database itself, that is, the time it takes for the query engine to receive a query from the client, fetch the data from the storage server, and perform a series of calculations.\nThe second number 1934 shows the time spent from the client's perspective, that is, the time it takes for the client from sending a request, receiving a response, and displaying the result on the screen.\n\"Why does the port number of the nebula-storaged process keep showing red after connecting to NebulaGraph?\"\nBecause the nebula-storaged process waits for nebula-metad to add the current Storage service during the startup process. The Storage works after it receives the ready signal. Starting from NebulaGraph 3.0.0, the Meta service cannot directly read or write data in the Storage service that you add in the configuration file. The configuration file only registers the Storage service to the Meta service. You must run the ADD HOSTS command to enable the Meta to read and write data in the Storage service. For more information, see Manage Storage hosts.\n\"Why is there no line separating each row in the returned result of NebulaGraph 2.6.0?\"\nThis is caused by the release of NebulaGraph Console 2.6.0, not the change of NebulaGraph core. And it will not affect the content of the returned data itself.\nAbout dangling edges\nA dangling edge is an edge that only connects to a single vertex and only one part of the edge connects to the vertex.\nDangling edges may appear in NebulaGraph 3.6.0 as the design. And there is no MERGE statements of openCypher. The guarantee for dangling edges depends entirely on the application level. For more information, see INSERT VERTEX, DELETE VERTEX, INSERT EDGE, DELETE EDGE.\n\"Can I set replica_factor as an even number in CREATE SPACE statements, e.g., replica_factor = 2?\"\nNO.\nThe Storage service guarantees its availability based on the Raft consensus protocol. The number of failed replicas must not exceed half of the total replica number.\nWhen the number of machines is 1, replica_factor can only be set to1.\nWhen there are enough machines and replica_factor=2, if one replica fails, the Storage service fails. No matter replica_factor=3 or replica_factor=4, if more than one replica fails, the Storage Service fails. To prevent unnecessary waste of resources, we recommend that you set an odd replica number.\nWe suggest that you set replica_factor=3 for a production environment and replica_factor=1 for a test environment. Do not use an even number.\n\"Is stopping or killing slow queries supported?\"\nYes. For more information, see Kill query.\n\"Why are the query results different when using GO and MATCH to execute the same semantic query?\"\nThe possible reasons are listed as follows.\nGO statements find the dangling edges.\nRETURN commands do not specify the sequence.\nThe dense vertex truncation limitation defined by max_edge_returned_per_vertex in the Storage service is triggered.\nUsing different types of paths may cause different query results.\nGO statements use walk. Both vertices and edges can be repeatedly visited in graph traversal.\nMATCH statements are compatible with openCypher and use trail. Only vertices can be repeatedly visited in graph traversal.\nThe example is as follows.\nAll queries that start from A with 5 hops will end at C (A->B->C->D->E->C). If it is 6 hops, the GO statement will end at D (A->B->C->D->E->C->D), because the edge C->D can be visited repeatedly. However, the MATCH statement returns empty, because edges cannot be visited repeatedly.\nTherefore, using GO and MATCH to execute the same semantic query may cause different query results.\nFor more information, see Wikipedia.\n\"How to count the vertices/edges number of each tag/edge type?\"\nSee show-stats.\n\"How to get all the vertices/edge of each tag/edge type?\"\nCreate and rebuild the index.\n> CREATE TAG INDEX IF NOT EXISTS i_player ON player();\n> REBUILD TAG INDEX IF NOT EXISTS i_player;\nUse LOOKUP or MATCH. For example:\n> LOOKUP ON player;\n> MATCH (n:player) RETURN n;\nFor more information, see INDEX, LOOKUP, and MATCH.\n\"Can non-English characters be used as identifiers, such as the names of graph spaces, tags, edge types, properties, and indexes?\"\nYes, for more information, see Keywords and reserved words.\n\"How to get the out-degree/the in-degree of a given vertex?\"\nThe out-degree of a vertex refers to the number of edges starting from that vertex, while the in-degree refers to the number of edges pointing to that vertex.\nnebula > MATCH (s)-[e]->() WHERE id(s) == \"given\" RETURN count(e); #Out-degree\nnebula > MATCH (s)<-[e]-() WHERE id(s) == \"given\" RETURN count(e); #In-degree\nThis is a very slow operation to get the out/in degree since no accelaration can be applied (no indices or caches). It also could be out-of-memory when hitting a supper-node.\n\"How to quickly get the out-degree and in-degree of all vertices?\"\nThere is no such command.\nYou can use NebulaGraph Algorithm.\nAbout operation and maintenance\n\"The runtime log files are too large. How to recycle the logs?\"\nBy default, the runtime logs of NebulaGraph are stored in /usr/local/nebula/logs/. The INFO level log files are nebula-graphd.INFO, nebula-storaged.INFO, nebula-metad.INFO. If an alarm or error occurs, the suffixes are modified as .WARNING or .ERROR.\nNebulaGraph uses glog to print logs. glog cannot recycle the outdated files. To rotate logs, you can:\nAdd the parameters timestamp_in_logfile_name=true (timestamp added to logfile) and max_log_size=500 (log size limit in MB) to the configuration files of the three services, and then use crontab to delete logs periodically. For more information, see Glog should delete old log files automatically.\nUse logrotate to manage log files. Before using logrotate, modify the configurations of corresponding services and set timestamp_in_logfile_name to false.\n\"How to check the NebulaGraph version?\"\nIf the service is running: run command SHOW HOSTS META in nebula-console. See SHOW HOSTS.\nIf the service is not running:\nDifferent installation methods make the method of checking the version different. The instructions are as follows:\nIf the service is not running, run the command ./ --version to get the version and the Git commit IDs of the NebulaGraph binary files. For example:\n$ ./nebula-graphd --version\nIf you deploy NebulaGraph with Docker Compose\nCheck the version of NebulaGraph deployed by Docker Compose. The method is similar to the previous method, except that you have to enter the container first. The commands are as follows:\ndocker exec -it nebula-docker-compose_graphd_1 bash\ncd bin/\n./nebula-graphd --version\nIf you install NebulaGraph with RPM/DEB package\nRun rpm -qa |grep nebula to check the version of NebulaGraph.\n\"How to scale my cluster up/down or out/in?\"\nIncrease or decrease the number of Meta, Graph, or Storage nodes\nNebulaGraph 3.6.0 does not provide any commands or tools to support automatic scale out/in. You can refer to the following steps:\nScale out and scale in metad: The metad process can not be scaled out or scale in. The process cannot be moved to a new machine. You cannot add a new metad process to the service.\nScale in graphd: Remove the IP of the graphd process from the code in the client. Close this graphd process.\nScale out graphd: Prepare the binary and config files of the graphd process in the new host. Modify the config files and add all existing addresses of the metad processes. Then start the new graphd process.\nScale in storaged: See Balance remove command. After the command is finished, stop this storaged process.\nScale out storaged: Prepare the binary and config files of the storaged process in the new host, Modify the config files and add all existing addresses of the metad processes. Then register the storaged process to the metad, and then start the new storaged process. For details, see Register storaged services.\nYou also need to run Balance Data and Balance leader after scaling in/out storaged.\nYou can scale Graph and Storage services with Dashboard Enterprise Edition. For details, see Scale. \nYou can also use NebulaGraph Operator to scale Graph and Storage services. For details, see Deploy NebulaGraph clusters.\nAdd or remove disks in the Storage nodes\nCurrently, Storage cannot dynamically recognize new added disks. You can add or remove disks in the Storage nodes of the distributed cluster by following these steps:\nExecute SUBMIT JOB BALANCE DATA REMOVE to migrate data in the Storage node with the disk to be added or removed to other Storage nodes.\nExecute DROP HOSTS to remove the Storage node with the disk to be added or removed.\nIn the configuration file of all Storage nodes, configure the path of the new disk to be added or removed through --data_path, see Storage configuration file for details.\nExecute ADD HOSTS to re-add the Storage node with the disk to be added or removed.\nAs needed, execute SUBMIT JOB BALANCE DATA to evenly distribute the shards of the current space to all Storage nodes and execute SUBMIT JOB BALANCE LEADER command to balance the leaders in all spaces. Before running the command, select a space.\n\"After changing the name of the host, the old one keeps displaying OFFLINE. What should I do?\"\nHosts with the status of OFFLINE will be automatically deleted after one day.\n\"How do I view the dmp file?\"\nThe dmp file is an error report file detailing the exit of the process and can be viewed with the gdb utility. the Coredump file is saved in the directory of the startup binary (by default it is /usr/local/nebula) and is generated automatically when the NebulaGraph service crashes.\nCheck the Core file process name, pid is usually a numeric value.\n $ file core.\nUse gdb to debug.\n $ gdb core.\nView the contents of the file.\n $(gdb) bt\nFor example:\n $ file core.1316027\ncore.1316027: ELF 64-bit LSB core file, x86-64, version 1 (SYSV), SVR4-style, from '/home/workspace/fork/nebula-debug/bin/nebula-metad --flagfile /home/k', real uid: 1008, effective uid: 1008, real gid: 1008, effective gid: 1008, execfn: '/home/workspace/fork/nebula-debug/bin/nebula-metad', platform: 'x86_64'\n$ gdb /home/workspace/fork/nebula-debug/bin/nebula-metad core.1316027\n$(gdb) bt\n#0 0x00007f9de58fecf5 in __memcpy_ssse3_back () from /lib64/libc.so.6\n#1 0x0000000000eb2299 in void std::__cxx11::basic_string, std::allocator >::_M_construct(char*, char*, std::forward_iterator_tag) ()\n#2 0x0000000000ef71a7 in nebula::meta::cpp2::QueryDesc::QueryDesc(nebula::meta::cpp2::QueryDesc const&) ()\n...\nIf you are not clear about the information that dmp prints out, you can post the printout with the OS version, hardware configuration, error logs before and after the Core file was created and actions that may have caused the error on the NebulaGraph forum.\nHow can I set the NebulaGraph service to start automatically on boot via systemctl?\nExecute systemctl enable to start the metad, graphd and storaged services.\n[root]# systemctl enable nebula-metad.service\nCreated symlink from /etc/systemd/system/multi-user.target.wants/nebula-metad.service to /usr/lib/systemd/system/nebula-metad.service.\n[root]# systemctl enable nebula-graphd.service\nCreated symlink from /etc/systemd/system/multi-user.target.wants/nebula-graphd.service to /usr/lib/systemd/system/nebula-graphd.service.\n[root]# systemctl enable nebula-storaged.service\nCreated symlink from /etc/systemd/system/multi-user.target.wants/nebula-storaged.service to /usr/lib/systemd/system/nebula-storaged.service.\nConfigure the service files for metad, graphd and storaged to set the service to pull up automatically.\nThe following points need to be noted when configuring the service file.\n - The paths of the PIDFile, ExecStart, ExecReload and ExecStop parameters need to be the same as those on the server.\n - RestartSec is the length of time (in seconds) to wait before restarting, which can be modified according to the actual situation.\n - (Optional) StartLimitInterval is the unlimited restart, the default is 10 seconds if the restart exceeds 5 times, and set to 0 means unlimited restart.\n - (Optional) LimitNOFILE is the maximum number of open files for the service, the default is 1024 and can be changed according to the actual situation.\nConfigure the service file for the metad service.\n$ vi /usr/lib/systemd/system/nebula-metad.service\n[Unit]\nDescription=Nebula Graph Metad Service\nAfter=network.target\n[Service ]\nType=forking\nRestart=always\nRestartSec=15s\nPIDFile=/usr/local/nebula/pids/nebula-metad.pid\nExecStart=/usr/local/nebula/scripts/nebula.service start metad\nExecReload=/usr/local/nebula/scripts/nebula.service restart metad\nExecStop=/usr/local/nebula/scripts/nebula.service stop metad\nPrivateTmp=true\nStartLimitInterval=0\nLimitNOFILE=1024\n[Install]\nWantedBy=multi-user.target\nConfigure the service file for the graphd service.\n$ vi /usr/lib/systemd/system/nebula-graphd.service\n[Unit]\nDescription=Nebula Graph Graphd Service\nAfter=network.target\n[Service]\nType=forking\nRestart=always\nRestartSec=15s\nPIDFile=/usr/local/nebula/pids/nebula-graphd.pid\nExecStart=/usr/local/nebula/scripts/nebula.service start graphd\nExecReload=/usr/local/nebula/scripts/nebula.service restart graphd\nExecStop=/usr/local/nebula/scripts/nebula.service stop graphd\nPrivateTmp=true\nStartLimitInterval=0\nLimitNOFILE=1024\n[Install]\nWantedBy=multi-user.target\n Configure the service file for the storaged service.\n$ vi /usr/lib/systemd/system/nebula-storaged.service\n[Unit]\nDescription=Nebula Graph Storaged Service\nAfter=network.target\n[Service]\nType=forking\nRestart=always\nRestartSec=15s\nPIDFile=/usr/local/nebula/pids/nebula-storaged.pid\nExecStart=/usr/local/nebula/scripts/nebula.service start storaged\nExecReload=/usr/local/nebula/scripts/nebula.service restart storaged\nExecStop=/usr/local/nebula/scripts/nebula.service stop storaged\nPrivateTmp=true\nStartLimitInterval=0\nLimitNOFILE=1024\n[Install]\nWantedBy=multi-user.target\nReload the configuration file.\n[root]# sudo systemctl daemon-reload\nRestart the service.\n$ systemctl restart nebula-metad.service\n$ systemctl restart nebula-graphd.service\n$ systemctl restart nebula-storaged.service\nAbout connections\n\"Which ports should be opened on the firewalls?\"\nIf you have not modified the predefined ports in the Configurations, open the following ports for the NebulaGraph services:\nService\nPort\nMeta\n9559, 9560, 19559\nGraph\n9669, 19669\nStorage\n9777 ~ 9780, 19779\nIf you have customized the configuration files and changed the predefined ports, find the port numbers in your configuration files and open them on the firewalls.\nFor more port information, see Port Guide for Company Products.\n\"How to test whether a port is open or closed?\"\nYou can use telnet as follows to check for port status.\ntelnet \nFor example:\n// If the port is open:\n$ telnet 192.168.1.10 9669\nTrying 192.168.1.10...\nConnected to 192.168.1.10.\nEscape character is '^]'.\n// If the port is closed or blocked:\n$ telnet 192.168.1.10 9777\nTrying 192.168.1.10...\ntelnet: connect to address 192.168.1.10: Connection refused","url":"https://docs.nebula-graph.io/3.6.0/20.appendix/0.FAQ/","type":"doc"},{"title":"NebulaGraph 3.6.0 release notes","content":"Features\nEnhance the full-text index. #5567 #5575 #5577 #5580 #5584 #5587\nThe changes involved are listed below:\nThe original full-text indexing function has been changed from calling Elasticsearch's Term-level queries to Full text queries.\nIn addition to supporting wildcards, regulars, fuzzy matches, etc. (but the syntax has been changed), support for word splitting (relying on Elasticsearch's own word splitter) has been added, and the query results include scoring results. For more syntax, see official Elasticsearch documentation.\nEnhancements\nSupport variables when querying vertex id or property index in a match clause. #5486 #5553\nPerformance\nSupport parallel startup of RocksDB instances to speed up the startup of the Storage service. #5521\nOptimize the prefix search performance of the RocksDB iterator after the DeleteRange operation. #5525\nOptimize the appendLog sending logic to avoid impacting write performance when a follower is down. #5571\nOptimize the performance of the MATCH statement when querying for non-existent properties. #5634\nBug fixes\nDQL\nFix the crash of the Graph service when executing a single big query. #5619\nFix the crash of the Graph service when executing the Find All Path statement. #5621 #5640\nFix the bug that some expired data is not recycled at the bottom level. #5447 #5622\nFix the bug that adding a path variable in the MATCH statement causes the all() function push-down optimization to fail. #5631\nFix the bug in the MATCH statement that returns incorrect results when querying the self-loop by the shortest path. #5636\nFix the bug that deleting edges by pipe causes the Graph service to crash. #5645\nFix the bug in the MATCH statement that returns missing properties of edges when matching multiple hops. #5646\nOthers\nFix the bug of meta data inconsistency. #5517\nFix the bug that RocksDB ingest causes the leader lease to be invalid. #5534\nFix the error in the statistics logic of storage. #5547\nFix the bug that causes the web service to crash if a flag is set for an invalid request parameter. #5566\nFix the bug that too many logs are printed when listing sessions. #5618","url":"https://docs.nebula-graph.io/3.6.0/20.appendix/release-notes/nebula-comm-release-note/","type":"doc"},{"title":"NebulaGraph Studio release notes","content":"v3.8.0\nFeatures\nSupported the use of MySQL databases as backend storage.\nEnhancements\nSupported customizing the read and write parameters of the WebSocket.\nUsability\nSupported filtering tasks in the import task list based on the map space name.\nCompatibility\nSince the database table structure has changed, you need to set DB.AutoMigrate to true in the configuration file, and the system will automatically upgrade and adapt the existing historical data.\nIf the tables were created manually after you consulted our after-sales staff, please modify these tables manually: task_infos, task_effects, sketches, schema_snapshots, favorites, files, and datasources.\nFor example:\nALTER TABLE `task_infos` ADD COLUMN `b_id` CHAR(32) NOT NULL DEFAULT '';\nUPDATE TABLE `task_infos` SET `b_id` = `id`;\nCREATE UNIQUE INDEX `idx_task_infos_id` ON `task_infos`(`b_id`);\nALTER TABLE `task_effects` ADD COLUMN `b_id` CHAR(32) NOT NULL DEFAULT '';\nUPDATE TABLE `task_effects` SET `b_id` = `id`;\nCREATE UNIQUE INDEX `idx_task_effects_id` ON `task_effects`(`b_id`);\n...\nv3.7.0\nEnhancements\nSupported importing SFTP, Amazon S3 data files.\nThe import page is supported to configure more import parameters, such as concurrency, retries, etc.\nSupported re-running tasks.\nSupported saving tasks as drafts.\nSupported running Studio in a docker container on the ARM architecture.","url":"https://docs.nebula-graph.io/3.6.0/20.appendix/release-notes/studio-release-note/","type":"doc"},{"title":"NebulaGraph Dashboard Community Edition 3.6.0 release notes","content":"Community Edition 3.4.0\nFeature\nSupport the built-in dashboard.service script to manage the Dashboard services with one-click and view the Dashboard version.\nSupport viewing the configuration of Meta services.\nEnhancement\nAdjust the directory structure and simplify the deployment steps.\nDisplay the names of the monitoring metrics on the overview page of machine.\nOptimize the calculation of monitoring metrics such as num_queries, and adjust the display to time series aggregation.","url":"https://docs.nebula-graph.io/3.6.0/20.appendix/release-notes/dashboard-comm-release-note/","type":"doc"},{"title":"Ecosystem tools overview","content":"NebulaGraph Studio\nNebulaGraph Studio (Studio for short) is a graph database visualization tool that can be accessed through the Web. It can be used with NebulaGraph DBMS to provide one-stop services such as composition, data import, writing nGQL queries, and graph exploration. For details, see What is NebulaGraph Studio.\nNebulaGraph version\nStudio version\nv3.6.0\nv3.8.0\nNebulaGraph Dashboard Community Edition\nNebulaGraph Dashboard Community Edition (Dashboard for short) is a visualization tool for monitoring the status of machines and services in the NebulaGraph cluster. For details, see What is NebulaGraph Dashboard.\nNebulaGraph version\nDashboard Community version\nv3.6.0\nv3.4.0\nNebulaGraph Exchange\nNebulaGraph Exchange (Exchange for short) is an Apache Spark&trade application for batch migration of data in a cluster to NebulaGraph in a distributed environment. It can support the migration of batch data and streaming data in a variety of different formats. For details, see What is NebulaGraph Exchange.\nNebulaGraph version\nExchange Community version\nv3.6.0\nv3.6.0\nNebulaGraph Operator\nNebulaGraph Operator (Operator for short) is a tool to automate the deployment, operation, and maintenance of NebulaGraph clusters on Kubernetes. Building upon the excellent scalability mechanism of Kubernetes, NebulaGraph introduced its operation and maintenance knowledge into the Kubernetes system, which makes NebulaGraph a real cloud-native graph database. For more information, see What is NebulaGraph Operator.\nNebulaGraph version\nOperator version\nv3.6.0\nv1.7.3\nNebulaGraph Importer\nNebulaGraph Importer (Importer for short) is a CSV file import tool for NebulaGraph. The Importer can read the local CSV file, and then import the data into the NebulaGraph database. For details, see What is NebulaGraph Importer.\nNebulaGraph version\nImporter version\nv3.6.0\nv4.1.0\nNebulaGraph Spark Connector\nNebulaGraph Spark Connector is a Spark connector that provides the ability to read and write NebulaGraph data in the Spark standard format. NebulaGraph Spark Connector consists of two parts, Reader and Writer. For details, see What is NebulaGraph Spark Connector.\nNebulaGraph version\nSpark Connector version\nv3.6.0\nv3.6.0\nNebulaGraph Flink Connector\nNebulaGraph Flink Connector is a connector that helps Flink users quickly access NebulaGraph. It supports reading data from the NebulaGraph database or writing data read from other external data sources to the NebulaGraph database. For details, see What is NebulaGraph Flink Connector.\nNebulaGraph version\nFlink Connector version\nv3.6.0\nv3.5.0\nNebulaGraph Algorithm\nNebulaGraph Algorithm (Algorithm for short) is a Spark application based on GraphX, which uses a complete algorithm tool to analyze data in the NebulaGraph database by submitting a Spark task To perform graph computing, use the algorithm under the lib repository through programming to perform graph computing for DataFrame. For details, see What is NebulaGraph Algorithm.\nNebulaGraph version\nAlgorithm version\nv3.6.0\nv3.0.0\nNebulaGraph Console\nNebulaGraph Console is the native CLI client of NebulaGraph. For how to use it, see NebulaGraph Console.\nNebulaGraph version\nConsole version\nv3.6.0\nv3.5.0\nNebulaGraph Docker Compose\nDocker Compose can quickly deploy NebulaGraph clusters. For how to use it, please refer to Docker Compose Deployment NebulaGraph.\nNebulaGraph version\nDocker Compose version\nv3.6.0\nv3.6.0\nBackup & Restore\nBackup&Restore (BR for short) is a command line interface (CLI) tool that can help back up the graph space data of NebulaGraph, or restore it through a backup file data.\nNebulaGraph version\nBR version\nv3.6.0\nv3.5.0\nNebulaGraph Bench\nNebulaGraph Bench is used to test the baseline performance data of NebulaGraph. It uses the standard data set of LDBC.\nNebulaGraph version\nBench version\nv3.6.0\nv1.2.0\nAPI, SDK\nNebulaGraph version\nLanguage\nv3.6.0\nC++\nv3.6.0\nGo\nv3.6.0\nPython\nv3.6.0\nJava\nv3.6.0\nHTTP\nNot Released\nRust Client\nNode.js Client\nObject Graph Mapping Library (OGM, or ORM) ","url":"https://docs.nebula-graph.io/3.6.0/20.appendix/6.eco-tool-version/","type":"doc"},{"title":"Port guide for company products","content":"The following are the default ports used by NebulaGraph core and peripheral tools.\nNo.\nProduct / Service\nType\nDefault\nDescription\nNebulaGraph\nTCP\n9669\nGraph service RPC daemon listening port. Commonly used for client connections to the Graph service.\nNebulaGraph\nTCP\n19669\nGraph service HTTP port.\nNebulaGraph\nTCP\n19670\nGraph service HTTP/2 port. (Deprecated after version 3.x)\nNebulaGraph\nTCP\n9559, 9560\n9559 is the RPC daemon listening port for Meta service. Commonly used by Graph and Storage services for querying and updating metadata in the graph database. The neighboring +1 (9560) port is used for Raft communication between Meta services.\nNebulaGraph\nTCP\n19559\nMeta service HTTP port.\nNebulaGraph\nTCP\n19560\nMeta service HTTP/2 port. (Deprecated after version 3.x)\nNebulaGraph\nTCP\n9779, 9778, 9780\n9779 is the RPC daemon listening port for Storage service. Commonly used by Graph services for data storage-related operations, such as reading, writing, or deleting data. The neighboring ports -1 (9778) and +1 (9780) are also used. 9778: The port used by the Admin service, which receives Meta commands for Storage. 9780: The port used for Raft communication between Storage services.\nNebulaGraph\nTCP\n19779\nStorage service HTTP port.\nNebulaGraph\nTCP\n19780\nStorage service HTTP/2 port. (Deprecated after version 3.x)\n10\nNebulaGraph\nTCP\n8888\nBackup and restore Agent service port. The Agent is a daemon running on each machine in the cluster, responsible for starting and stopping NebulaGraph services and uploading and downloading backup files.\n11\nNebulaGraph\nTCP\n9789, 9788, 9790\n9789 is the Raft Listener port for Full-text index, which reads data from Storage services and writes it to the Elasticsearch cluster.Also the port for Storage Listener in inter-cluster data synchronization, used for synchronizing Storage data from the primary cluster. The neighboring ports -1 (9788) and +1 (9790) are also used.9788: An internal port.9790: The port used for Raft communication.\n12\nNebulaGraph\nTCP\n9200\nNebulaGraph uses this port for HTTP communication with Elasticsearch to perform full-text search queries and manage full-text indexes.\n13\nNebulaGraph\nTCP\n9569, 9568, 9570\n9569 is the Meta Listener port in inter-cluster data synchronization, used for synchronizing Meta data from the primary cluster. The neighboring ports -1 (9568) and +1 (9570) are also used.9568: An internal port.9570: The port used for Raft communication.\n14\nNebulaGraph\nTCP\n9889, 9888, 9890\nDrainer service port in inter-cluster data synchronization, used for synchronizing Storage and Meta data to the primary cluster. The neighboring ports -1 (9888) and +1 (9890) are also used.9888: An internal port.9890: The port used for Raft communication.\n15\nNebulaGraph Studio\nTCP\n7001\nStudio web service port.\n16\nNebulaGraph Dashboard\nTCP\n8090\nNebula HTTP Gateway dependency service port. Provides an HTTP interface for cluster services to interact with the NebulaGraph database using nGQL statements.0\n17\nNebulaGraph Dashboard\nTCP\n9200\nNebula Stats Exporter dependency service port. Collects cluster performance metrics, including service IP addresses, versions, and monitoring metrics (such as query count, query latency, heartbeat latency, etc.).\n18\nNebulaGraph Dashboard\nTCP\n9100\nNode Exporter dependency service port. Collects resource information for machines in the cluster, including CPU, memory, load, disk, and traffic.\n19\nNebulaGraph Dashboard\nTCP\n9090\nPrometheus service port. Time-series database for storing monitoring data.\n20\nNebulaGraph Dashboard\nTCP\n7003\nDashboard Community Edition web service port.","url":"https://docs.nebula-graph.io/3.6.0/20.appendix/port-guide/","type":"doc"},{"title":"How to Contribute","content":"Before you get started\nCommit an issue on the github or forum\nYou are welcome to contribute any code or files to the project. But firstly we suggest you raise an issue on the github or the forum to start a discussion with the community. Check through the topic for Github.\nSign the Contributor License Agreement (CLA)\nWhat is CLA?\nHere is the vesoft inc. Contributor License Agreement.\nClick the Sign in with GitHub to agree button to sign the CLA.\nIf you have any questions, send an email to info@vesoft.com.\nModify a single document\nThis manual is written in the Markdown language. Click the pencil icon on the right of the document title to commit the modification.\nThis method applies to modify a single document only.\nBatch modify or add files\nThis method applies to contribute codes, modify multiple documents in batches, or add new documents.\nStep 1: Fork in the github.com\nThe NebulaGraph project has many repositories. Take the nebul repository for example:\nVisit https://github.com/vesoft-inc/nebula.\nClick the Fork button to establish an online fork.\nStep 2: Clone Fork to Local Storage\nDefine a local working directory.\n# Define the working directory.\nworking_dir=$HOME/Workspace\nSet user to match the Github profile name.\nuser={the Github profile name}\nCreate your clone.\nmkdir -p $working_dir\ncd $working_dir\ngit clone https://github.com/$user/nebula.git\n# or: git clone git@github.com:$user/nebula.git\ncd $working_dir/nebula\ngit remote add upstream https://github.com/vesoft-inc/nebula.git\n# or: git remote add upstream git@github.com:vesoft-inc/nebula.git\n# Never push to upstream master since you do not have write access.\ngit remote set-url --push upstream no_push\n# Confirm that the remote branch is valid.\n# The correct format is:\n# origin git@github.com:$(user)/nebula.git (fetch)\n# origin git@github.com:$(user)/nebula.git (push)\n# upstream https://github.com/vesoft-inc/nebula (fetch)\n# upstream no_push (push)\ngit remote -v\n(Optional) Define a pre-commit hook.\nPlease link the NebulaGraph pre-commit hook into the .git directory.\nThis hook checks the commits for formatting, building, doc generation, etc.\ncd $working_dir/nebula/.git/hooks\nln -s $working_dir/nebula/.linters/cpp/hooks/pre-commit.sh .\nSometimes, the pre-commit hook cannot be executed. You have to execute it manually.\ncd $working_dir/nebula/.git/hooks\nchmod +x pre-commit\nStep 3: Branch\nGet your local master up to date.\ncd $working_dir/nebula\ngit fetch upstream\ngit checkout master\ngit rebase upstream/master\nCheckout a new branch from master.\ngit checkout -b myfeature\nStep 4: Develop\nCode style\nNebulaGraph adopts cpplint to make sure that the project conforms to Google's coding style guides. The checker will be implemented before the code is committed.\nUnit tests requirements\nPlease add unit tests for the new features or bug fixes.\nBuild your code with unit tests enabled\nFor more information, see Install NebulaGraph by compiling the source code.\nRun tests\nIn the root directory of nebula, run the following command:\ncd nebula/build\nctest -j$(nproc)\nStep 5: Bring Your Branch Update to Date\n# While on your myfeature branch.\ngit fetch upstream\ngit rebase upstream/master\nUsers need to bring the head branch up to date after other contributors merge PR to the base branch.\nStep 6: Commit\nCommit your changes.\ngit commit -a\nUsers can use the command --amend to re-edit the previous code.\nStep 7: Push\nWhen ready to review or just to establish an offsite backup, push your branch to your fork on github.com:\ngit push origin myfeature\nStep 8: Create a Pull Request\nVisit your fork at https://github.com/$user/nebula (replace $user here).\nClick the Compare & pull request button next to your myfeature branch.\nStep 9: Get a Code Review\nOnce your pull request has been created, it will be assigned to at least two reviewers. Those reviewers will do a thorough code review to make sure that the changes meet the repository's contributing guidelines and other quality standards.\nAdd test cases\nFor detailed methods, see How to add test cases.\nDonation\nStep 1: Confirm the project donation\nContact the official NebulaGraph staff via email, WeChat, Slack, etc. to confirm the donation project. The project will be donated to the NebulaGraph Contrib organization.\nEmail address: info@vesoft.com\nWeChat: NebulaGraphbot\nSlack: Join Slack\nStep 2: Get the information of the project recipient\nThe NebulaGraph official staff will give the recipient ID of the NebulaGraph Contrib project.\nStep 3: Donate a project\nThe user transfers the project to the recipient of this donation, and the recipient transfers the project to the NebulaGraph Contrib organization. After the donation, the user will continue to lead the development of community projects as a Maintainer.\nFor operations of transferring a repository on GitHub, see Transferring a repository owned by your user account.","url":"https://docs.nebula-graph.io/3.6.0/15.contribution/how-to-contribute/","type":"doc"},{"title":"History timeline for NebulaGraph","content":"2018.9: dutor wrote and submitted the first line of NebulaGraph database code. \n2019.5: NebulaGraph v0.1.0-alpha was released as open-source.\nNebulaGraph v1.0.0-beta, v1.0.0-rc1, v1.0.0-rc2, v1.0.0-rc3, and v1.0.0-rc4 were released one after another within a year thereafter.\n2019.7: NebulaGraph's debut at HBaseCon1. @dangleptr\n2020.3: NebulaGraph v2.0 was starting developed in the final stage of v1.0 development. \n2020.6: The first major version of NebulaGraph v1.0.0 GA was released.\n2021.3: The second major version of NebulaGraph v2.0 GA was released.\n2021.8: NebulaGraph v2.5.0 was released.\n2021.10: NebulaGraph v2.6.0 was released.\n2022.2: NebulaGraph v3.0.0 was released.\n2022.4: NebulaGraph v3.1.0 was released.\n2022.7: NebulaGraph v3.2.0 was released.\n2022.10: NebulaGraph v3.3.0 was released.\n2023.2: NebulaGraph v3.4.0 was released.\n2023.5: NebulaGraph v3.5.0 was released.\n2023.8: NebulaGraph v3.6.0 was released.\nNebulaGraph v1.x supports both RocksDB and HBase as its storage engines. NebulaGraph v2.x removes HBase supports. ↩","url":"https://docs.nebula-graph.io/3.6.0/20.appendix/history/","type":"doc"},{"title":"Error code","content":"NebulaGraph returns an error code when an error occurs. This topic describes the details of the error code returned. \nError name\nError Code\nDescription\nE_DISCONNECTED\n-1\nLost connection\nE_FAIL_TO_CONNECT\n-2\nUnable to establish connection\nE_RPC_FAILURE\n-3\nRPC failure\nE_LEADER_CHANGED\n-4\nRaft leader has been changed\nE_SPACE_NOT_FOUND\n-5\nGraph space does not exist\nE_TAG_NOT_FOUND\n-6\nTag does not exist\nE_EDGE_NOT_FOUND\n-7\nEdge type does not exist\nE_INDEX_NOT_FOUND\n-8\nIndex does not exist\nE_EDGE_PROP_NOT_FOUND\n-9\nEdge type property does not exist\nE_TAG_PROP_NOT_FOUND\n-10\nTag property does not exist\nE_ROLE_NOT_FOUND\n-11\nThe current role does not exist\nE_CONFIG_NOT_FOUND\n-12\nThe current configuration does not exist\nE_MACHINE_NOT_FOUND\n-13\nThe current host does not exist\nE_LISTENER_NOT_FOUND\n-15\nListener does not exist\nE_PART_NOT_FOUND\n-16\nThe current partition does not exist\nE_KEY_NOT_FOUND\n-17\nKey does not exist\nE_USER_NOT_FOUND\n-18\nUser does not exist\nE_STATS_NOT_FOUND\n-19\nStatistics do not exist\nE_SERVICE_NOT_FOUND\n-20\nNo current service found\nE_DRAINER_NOT_FOUND\n-21\nDrainer does not exist\nE_DRAINER_CLIENT_NOT_FOUND\n-22\nDrainer client does not exist\nE_PART_STOPPED\n-23\nThe current partition has already been stopped\nE_BACKUP_FAILED\n-24\nBackup failed\nE_BACKUP_EMPTY_TABLE\n-25\nThe backed-up table is empty\nE_BACKUP_TABLE_FAILED\n-26\nTable backup failure\nE_PARTIAL_RESULT\n-27\nMultiGet could not get all data\nE_REBUILD_INDEX_FAILED\n-28\nIndex rebuild failed\nE_INVALID_PASSWORD\n-29\nPassword is invalid\nE_FAILED_GET_ABS_PATH\n-30\nUnable to get absolute path\nE_BAD_USERNAME_PASSWORD\n-1001\nAuthentication failed\nE_SESSION_INVALID\n-1002\nInvalid session\nE_SESSION_TIMEOUT\n-1003\nSession timeout\nE_SYNTAX_ERROR\n-1004\nSyntax error\nE_EXECUTION_ERROR\n-1005\nExecution error\nE_STATEMENT_EMPTY\n-1006\nStatement is empty\nE_BAD_PERMISSION\n-1008\nPermission denied\nE_SEMANTIC_ERROR\n-1009\nSemantic error\nE_TOO_MANY_CONNECTIONS\n-1010\nMaximum number of connections exceeded\nE_PARTIAL_SUCCEEDED\n-1011\nAccess to storage failed (only some requests succeeded)\nE_NO_HOSTS\n-2001\nHost does not exist\nE_EXISTED\n-2002\nHost already exists\nE_INVALID_HOST\n-2003\nInvalid host\nE_UNSUPPORTED\n-2004\nThe current command, statement, or function is not supported\nE_NOT_DROP\n-2005\nNot allowed to drop\nE_CONFIG_IMMUTABLE\n-2007\nConfiguration items cannot be changed\nE_CONFLICT\n-2008\nParameters conflict with meta data\nE_INVALID_PARM\n-2009\nInvalid parameter\nE_WRONGCLUSTER\n-2010\nWrong cluster\nE_ZONE_NOT_ENOUGH\n-2011\nListener conflicts\nE_ZONE_IS_EMPTY\n-2012\nHost not exist\nE_SCHEMA_NAME_EXISTS\n-2013\nSchema name already exists\nE_RELATED_INDEX_EXISTS\n-2014\nThere are still indexes related to tag or edge, cannot drop it\nE_RELATED_SPACE_EXISTS\n-2015\nThere are still some space on the host, cannot drop it\nE_STORE_FAILURE\n-2021\nFailed to store data\nE_STORE_SEGMENT_ILLEGAL\n-2022\nIllegal storage segment\nE_BAD_BALANCE_PLAN\n-2023\nInvalid data balancing plan\nE_BALANCED\n-2024\nThe cluster is already in the data balancing status\nE_NO_RUNNING_BALANCE_PLAN\n-2025\nThere is no running data balancing plan\nE_NO_VALID_HOST\n-2026\nLack of valid hosts\nE_CORRUPTED_BALANCE_PLAN\n-2027\nA data balancing plan that has been corrupted\nE_IMPROPER_ROLE\n-2030\nFailed to recover user role\nE_INVALID_PARTITION_NUM\n-2031\nNumber of invalid partitions\nE_INVALID_REPLICA_FACTOR\n-2032\nInvalid replica factor\nE_INVALID_CHARSET\n-2033\nInvalid character set\nE_INVALID_COLLATE\n-2034\nInvalid character sorting rules\nE_CHARSET_COLLATE_NOT_MATCH\n-2035\nCharacter set and character sorting rule mismatch\nE_SNAPSHOT_FAILURE\n-2040\nFailed to generate a snapshot\nE_BLOCK_WRITE_FAILURE\n-2041\nFailed to write block data\nE_ADD_JOB_FAILURE\n-2044\nFailed to add new task\nE_STOP_JOB_FAILURE\n-2045\nFailed to stop task\nE_SAVE_JOB_FAILURE\n-2046\nFailed to save task information\nE_BALANCER_FAILURE\n-2047\nData balancing failed\nE_JOB_NOT_FINISHED\n-2048\nThe current task has not been completed\nE_TASK_REPORT_OUT_DATE\n-2049\nTask report failed\nE_JOB_NOT_IN_SPACE\n-2050\nThe current task is not in the graph space\nE_JOB_NEED_RECOVER\n-2051\nThe current task needs to be resumed\nE_JOB_ALREADY_FINISH\n-2052\nThe job status has already been failed or finished\nE_JOB_SUBMITTED\n-2053\nJob default status\nE_JOB_NOT_STOPPABLE\n-2054\nThe given job do not support stop\nE_JOB_HAS_NO_TARGET_STORAGE\n-2055\nThe leader distribution has not been reported, so can't send task to storage\nE_INVALID_JOB\n-2065\nInvalid task\nE_BACKUP_BUILDING_INDEX\n-2066\nBackup terminated (index being created)\nE_BACKUP_SPACE_NOT_FOUND\n-2067\nGraph space does not exist at the time of backup\nE_RESTORE_FAILURE\n-2068\nBackup recovery failed\nE_SESSION_NOT_FOUND\n-2069\nSession does not exist\nE_LIST_CLUSTER_FAILURE\n-2070\nFailed to get cluster information\nE_LIST_CLUSTER_GET_ABS_PATH_FAILURE\n-2071\nFailed to get absolute path when getting cluster information\nE_LIST_CLUSTER_NO_AGENT_FAILURE\n-2072\nUnable to get an agent when getting cluster information\nE_QUERY_NOT_FOUND\n-2073\nQuery not found\nE_AGENT_HB_FAILUE\n-2074\nFailed to receive heartbeat from agent\nE_HOST_CAN_NOT_BE_ADDED\n-2082\nThe host can not be added for it's not a storage host\nE_ACCESS_ES_FAILURE\n-2090\nFailed to access elasticsearch\nE_GRAPH_MEMORY_EXCEEDED\n-2600\nGraph memory exceeded\nE_CONSENSUS_ERROR\n-3001\nConsensus cannot be reached during an election\nE_KEY_HAS_EXISTS\n-3002\nKey already exists\nE_DATA_TYPE_MISMATCH\n-3003\nData type mismatch\nE_INVALID_FIELD_VALUE\n-3004\nInvalid field value\nE_INVALID_OPERATION\n-3005\nInvalid operation\nE_NOT_NULLABLE\n-3006\nCurrent value is not allowed to be empty\nE_FIELD_UNSET\n-3007\nField value must be set if the field value is NOT NULL or has no default value\nE_OUT_OF_RANGE\n-3008\nThe value is out of the range of the current type\nE_DATA_CONFLICT_ERROR\n-3010\nData conflict\nE_WRITE_STALLED\n-3011\nWrites are delayed\nE_IMPROPER_DATA_TYPE\n-3021\nIncorrect data type\nE_INVALID_SPACEVIDLEN\n-3022\nInvalid VID length\nE_INVALID_FILTER\n-3031\nInvalid filter\nE_INVALID_UPDATER\n-3032\nInvalid field update\nE_INVALID_STORE\n-3033\nInvalid KV storage\nE_INVALID_PEER\n-3034\nPeer invalid\nE_RETRY_EXHAUSTED\n-3035\nOut of retries\nE_TRANSFER_LEADER_FAILED\n-3036\nLeader change failed\nE_INVALID_STAT_TYPE\n-3037\nInvalid stat type\nE_INVALID_VID\n-3038\nVID is invalid\nE_LOAD_META_FAILED\n-3040\nFailed to load meta information\nE_FAILED_TO_CHECKPOINT\n-3041\nFailed to generate checkpoint\nE_CHECKPOINT_BLOCKED\n-3042\nGenerating checkpoint is blocked\nE_FILTER_OUT\n-3043\nData is filtered\nE_INVALID_DATA\n-3044\nInvalid data\nE_MUTATE_EDGE_CONFLICT\n-3045\nConcurrent write conflicts on the same edge\nE_MUTATE_TAG_CONFLICT\n-3046\nConcurrent write conflict on the same vertex\nE_OUTDATED_LOCK\n-3047\nLock is invalid\nE_INVALID_TASK_PARA\n-3051\nInvalid task parameter\nE_USER_CANCEL\n-3052\nThe user canceled the task\nE_TASK_EXECUTION_FAILED\n-3053\nTask execution failed\nE_PLAN_IS_KILLED\n-3060\nExecution plan was cleared\nE_NO_TERM\n-3070\nThe heartbeat process was not completed when the request was received\nE_OUTDATED_TERM\n-3071\nOut-of-date heartbeat received from the old leader (the new leader has been elected)\nE_WRITE_WRITE_CONFLICT\n-3073\nConcurrent write conflicts with later requests\nE_RAFT_UNKNOWN_PART\n-3500\nUnknown partition\nE_RAFT_LOG_GAP\n-3501\nRaft logs lag behind\nE_RAFT_LOG_STALE\n-3502\nRaft logs are out of date\nE_RAFT_TERM_OUT_OF_DATE\n-3503\nHeartbeat messages are out of date\nE_RAFT_UNKNOWN_APPEND_LOG\n-3504\nUnknown additional logs\nE_RAFT_WAITING_SNAPSHOT\n-3511\nWaiting for the snapshot to complete\nE_RAFT_SENDING_SNAPSHOT\n-3512\nThere was an error sending the snapshot\nE_RAFT_INVALID_PEER\n-3513\nInvalid receiver\nE_RAFT_NOT_READY\n-3514\nRaft did not start\nE_RAFT_STOPPED\n-3515\nRaft has stopped\nE_RAFT_BAD_ROLE\n-3516\nWrong role\nE_RAFT_WAL_FAIL\n-3521\nWrite to a WAL failed\nE_RAFT_HOST_STOPPED\n-3522\nThe host has stopped\nE_RAFT_TOO_MANY_REQUESTS\n-3523\nToo many requests\nE_RAFT_PERSIST_SNAPSHOT_FAILED\n-3524\nPersistent snapshot failed\nE_RAFT_RPC_EXCEPTION\n-3525\nRPC exception\nE_RAFT_NO_WAL_FOUND\n-3526\nNo WAL logs found\nE_RAFT_HOST_PAUSED\n-3527\nHost suspended\nE_RAFT_WRITE_BLOCKED\n-3528\nWrites are blocked\nE_RAFT_BUFFER_OVERFLOW\n-3529\nCache overflow\nE_RAFT_ATOMIC_OP_FAILED\n-3530\nAtomic operation failed\nE_LEADER_LEASE_FAILED\n-3531\nLeader lease expired\nE_RAFT_CAUGHT_UP\n-3532\nData has been synchronized on Raft\nE_STORAGE_MEMORY_EXCEEDED\n-3600\nStorage memory exceeded\nE_LOG_GAP\n-4001\nDrainer logs lag behind\nE_LOG_STALE\n-4002\nDrainer logs are out of date\nE_INVALID_DRAINER_STORE\n-4003\nThe drainer data storage is invalid\nE_SPACE_MISMATCH\n-4004\nGraph space mismatch\nE_PART_MISMATCH\n-4005\nPartition mismatch\nE_DATA_CONFLICT\n-4006\nData conflict\nE_REQ_CONFLICT\n-4007\nRequest conflict\nE_DATA_ILLEGAL\n-4008\nIllegal data\nE_CACHE_CONFIG_ERROR\n-5001\nCache configuration error\nE_NOT_ENOUGH_SPACE\n-5002\nInsufficient space\nE_CACHE_MISS\n-5003\nNo cache hit\nE_CACHE_WRITE_FAILURE\n-5005\nWrite cache failed\nE_NODE_NUMBER_EXCEED_LIMIT\n-7001\nNumber of machines exceeded the limit\nE_PARSING_LICENSE_FAILURE\n-7002\nFailed to resolve certificate\nE_UNKNOWN\n-8000\nUnknown error","url":"https://docs.nebula-graph.io/3.6.0/20.appendix/error-code/","type":"doc"}] \ No newline at end of file diff --git a/app/utils/ngql.ts b/app/utils/ngql.ts new file mode 100644 index 00000000..69065497 --- /dev/null +++ b/app/utils/ngql.ts @@ -0,0 +1,31 @@ +import ngqlJson from './ngql.json'; + +const urlTransformerMap = { + FETCH: 'FETCHProps', + MATCH: 'MatchOrCypherOrPatternOrScan', + GO: 'GOFromVID', + FIND: 'FindPath', +}; +export const ngqlDoc = (ngqlJson as { url: string; content: string; title: string }[]) + .map((item) => { + if (urlTransformerMap[item.title]) { + item.title = urlTransformerMap[item.title]; + } + item.title = item.title.replaceAll(' ', ''); + item.content = item.content.replace(/nebula>/g, ''); + + return item; + }) + .filter((item) => { + return item.url.indexOf('clauses-and-options/') === -1; + }); +export const ngqlMap = ngqlDoc.reduce((acc, item) => { + acc[item.title.toLowerCase()] = item; + return acc; +}); +//@ts-ignore +window.ngqlMap = ngqlMap; +export const NGQLCategoryString = ngqlDoc + .filter((item) => item.url.indexOf('ngql-guide') >= 0) + .map((item) => item.title) + .join(','); diff --git a/app/utils/websocket.ts b/app/utils/websocket.ts index f18bfad5..04976370 100644 --- a/app/utils/websocket.ts +++ b/app/utils/websocket.ts @@ -218,7 +218,9 @@ export class NgqlRunner { message.error(content.message); } messageReceiver.resolve(content); - this.messageReceiverMap.delete(msgReceive.header.msgId); + if (messageReceiver?.config?.notClear !== true) { + this.messageReceiverMap.delete(msgReceive.header.msgId); + } } }; @@ -311,6 +313,30 @@ export class NgqlRunner { this.messageReceiverMap.set(messageReceiver.messageSend.header.msgId, messageReceiver); }); }; + + runChat = async ({ req, callback }: { req: any; callback?: (str: any) => void }, config: Recordable = {}) => { + if (!this.socket || this.socket.readyState !== WebSocket.OPEN) { + await this.reConnect(); + } + + return new Promise((resolve, reject) => { + if (req.stream) { + resolve(req.stream); + } + const messageReceiver = new MessageReceiver({ + resolve: req.stream ? callback : resolve, // when stream don't use promise to reveive data + reject, + product: this.product, + content: req, + config, + msgType: 'llm', + }); + config['notClear'] = true; + + this.socket.send(JSON.stringify(messageReceiver.messageSend)); + this.messageReceiverMap.set(messageReceiver.messageSend.header.msgId, messageReceiver); + }); + }; } const ngqlRunner = new NgqlRunner(); diff --git a/go.work b/go.work new file mode 100644 index 00000000..c3bb04b0 --- /dev/null +++ b/go.work @@ -0,0 +1,5 @@ +go 1.21 + +use ( + ./server +) diff --git a/package-lock.json b/package-lock.json index e98c5c69..ee162565 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "lodash.debounce": "^4.0.8", "mobx": "^6.9.0", "mobx-react-lite": "^3.4.3", + "pdfjs-dist": "^4.0.269", "query-string": "^7.1.1", "react": "^18.2.0", "react-beautiful-dnd": "^13.1.1", @@ -66,6 +67,7 @@ "@typescript-eslint/parser": "^5.0.0", "@vitejs/plugin-react-swc": "^3.3.1", "autoprefixer": "^10.4.14", + "cheerio": "^1.0.0-rc.12", "ejs": "^3.1.9", "eslint": "^8.41.0", "eslint-config-prettier": "^8.8.0", @@ -75,7 +77,7 @@ "less": "^4.1.1", "postcss": "^8.4.12", "postcss-preset-env": "^8.4.1", - "prettier": "^2.3.2", + "prettier": "^2.8.8", "react-dev-utils": "^12.0.0", "stylelint": "^15.6.2", "stylelint-config-recommended": "^12.0.0", @@ -1236,6 +1238,26 @@ "react": "*" } }, + "node_modules/@mapbox/node-pre-gyp": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", + "integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.0", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.7", + "nopt": "^5.0.0", + "npmlog": "^5.0.1", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.11" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + } + }, "node_modules/@monaco-editor/loader": { "version": "1.4.0", "resolved": "https://registry.npmmirror.com/@monaco-editor/loader/-/loader-1.4.0.tgz", @@ -1854,21 +1876,6 @@ } } }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/semver": { - "version": "7.3.7", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", - "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@typescript-eslint/parser": { "version": "5.59.7", "resolved": "https://registry.npmmirror.com/@typescript-eslint/parser/-/parser-5.59.7.tgz", @@ -1960,21 +1967,6 @@ } } }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { - "version": "7.5.1", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.5.1.tgz", - "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", - "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@typescript-eslint/utils": { "version": "5.59.7", "resolved": "https://registry.npmmirror.com/@typescript-eslint/utils/-/utils-5.59.7.tgz", @@ -1997,21 +1989,6 @@ "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/@typescript-eslint/utils/node_modules/semver": { - "version": "7.5.1", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.5.1.tgz", - "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", - "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@typescript-eslint/visitor-keys": { "version": "5.59.7", "resolved": "https://registry.npmmirror.com/@typescript-eslint/visitor-keys/-/visitor-keys-5.59.7.tgz", @@ -2279,6 +2256,12 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "peer": true }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "optional": true + }, "node_modules/accessor-fn": { "version": "1.3.2", "resolved": "https://registry.npmmirror.com/accessor-fn/-/accessor-fn-1.3.2.tgz", @@ -2322,6 +2305,18 @@ "node": ">= 0.12.0" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "optional": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmmirror.com/ajv/-/ajv-6.12.6.tgz", @@ -2345,7 +2340,7 @@ "version": "5.0.1", "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -2434,6 +2429,25 @@ "node": ">= 8" } }, + "node_modules/aproba": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", + "optional": true + }, + "node_modules/are-we-there-yet": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", + "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", + "optional": true, + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmmirror.com/argparse/-/argparse-2.0.1.tgz", @@ -2526,7 +2540,7 @@ "version": "1.0.2", "resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "devOptional": true }, "node_modules/bezier-js": { "version": "4.1.1", @@ -2558,11 +2572,17 @@ "node": ">=8" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, + "devOptional": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2640,6 +2660,21 @@ "resolved": "https://registry.npmmirror.com/caniuse-lite/-/caniuse-lite-1.0.30001489.tgz", "integrity": "sha512-x1mgZEXK8jHIfAxm+xgdpHpk50IN3z3q3zP261/WS+uvePxW8izXuCu6AHz0lkuYTlATDehiZ/tNyYBdSQsOUQ==" }, + "node_modules/canvas": { + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz", + "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "@mapbox/node-pre-gyp": "^1.0.0", + "nan": "^2.17.0", + "simple-get": "^3.0.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/canvas-color-tracker": { "version": "1.1.5", "resolved": "https://registry.npmmirror.com/canvas-color-tracker/-/canvas-color-tracker-1.1.5.tgz", @@ -2678,6 +2713,38 @@ "node": ">=4" } }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmmirror.com/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dev": true, + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dev": true, + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz", @@ -2699,6 +2766,15 @@ "fsevents": "~2.3.2" } }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "optional": true, + "engines": { + "node": ">=10" + } + }, "node_modules/chrome-trace-event": { "version": "1.0.3", "resolved": "https://registry.npmmirror.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", @@ -2728,6 +2804,15 @@ "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "optional": true, + "bin": { + "color-support": "bin.js" + } + }, "node_modules/colord": { "version": "2.9.3", "resolved": "https://registry.npmmirror.com/colord/-/colord-2.9.3.tgz", @@ -2749,7 +2834,13 @@ "version": "0.0.1", "resolved": "https://registry.npmmirror.com/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true + "devOptional": true + }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "optional": true }, "node_modules/cookie": { "version": "0.3.1", @@ -2857,6 +2948,19 @@ "postcss": "^8.4" } }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dev": true, + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + } + }, "node_modules/css-tree": { "version": "2.3.1", "resolved": "https://registry.npmmirror.com/css-tree/-/css-tree-2.3.1.tgz", @@ -2870,6 +2974,15 @@ "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" } }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/cssdb": { "version": "7.6.0", "resolved": "https://registry.npmmirror.com/cssdb/-/cssdb-7.6.0.tgz", @@ -3080,7 +3193,7 @@ "version": "4.3.4", "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, + "devOptional": true, "dependencies": { "ms": "2.1.2" }, @@ -3097,7 +3210,7 @@ "version": "2.1.2", "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "devOptional": true }, "node_modules/decamelize": { "version": "1.2.0", @@ -3138,6 +3251,18 @@ "node": ">=0.10" } }, + "node_modules/decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "optional": true, + "dependencies": { + "mimic-response": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmmirror.com/deep-is/-/deep-is-0.1.4.tgz", @@ -3162,6 +3287,21 @@ "node": ">=8" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "optional": true + }, + "node_modules/detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/detect-port-alt": { "version": "1.1.6", "resolved": "https://registry.npmmirror.com/detect-port-alt/-/detect-port-alt-1.1.6.tgz", @@ -3223,11 +3363,51 @@ "resolved": "https://registry.npmmirror.com/dom-align/-/dom-align-1.12.4.tgz", "integrity": "sha512-R8LUSEay/68zE5c8/3BDxiTEvgb4xZTF0RKmAHfiEVN3klfIpXfi2/QCoiWPccVQ0J/ZGdz9OjzL4uJEP/MRAw==" }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dev": true, + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + } + }, "node_modules/dom-to-image": { "version": "2.6.0", "resolved": "https://registry.npmmirror.com/dom-to-image/-/dom-to-image-2.6.0.tgz", "integrity": "sha512-Dt0QdaHmLpjURjU7Tnu3AgYSF2LuOmksSGsUcE6ItvJoCWTBEmiMXcqBdNSAm9+QbbwD7JMoVsuuKX6ZVQv1qA==" }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmmirror.com/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dev": true, + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dev": true, + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + } + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmmirror.com/duplexer/-/duplexer-0.1.2.tgz", @@ -3258,7 +3438,7 @@ "version": "8.0.0", "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "devOptional": true }, "node_modules/emojis-list": { "version": "3.0.0", @@ -3281,6 +3461,15 @@ "node": ">=10.13.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmmirror.com/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "dev": true, + "engines": { + "node": ">=0.12" + } + }, "node_modules/errno": { "version": "0.1.8", "resolved": "https://registry.npmmirror.com/errno/-/errno-0.1.8.tgz", @@ -3945,18 +4134,6 @@ "node": "^10.12.0 || >=12.0.0" } }, - "node_modules/flat-cache/node_modules/rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dev": true, - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - } - }, "node_modules/flatted": { "version": "3.2.5", "resolved": "https://registry.npmmirror.com/flatted/-/flatted-3.2.5.tgz", @@ -4133,21 +4310,6 @@ "node": ">= 8.9.0" } }, - "node_modules/fork-ts-checker-webpack-plugin/node_modules/semver": { - "version": "7.3.7", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", - "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/fork-ts-checker-webpack-plugin/node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", @@ -4178,6 +4340,30 @@ "node": "*" } }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/fs-monkey": { "version": "1.0.3", "resolved": "https://registry.npmmirror.com/fs-monkey/-/fs-monkey-1.0.3.tgz", @@ -4188,7 +4374,7 @@ "version": "1.0.0", "resolved": "https://registry.npmmirror.com/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "devOptional": true }, "node_modules/fsevents": { "version": "2.3.2", @@ -4210,6 +4396,26 @@ "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true }, + "node_modules/gauge": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "optional": true, + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.2", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.1", + "object-assign": "^4.1.1", + "signal-exit": "^3.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.2" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/gl-matrix": { "version": "3.4.3", "resolved": "https://registry.npmmirror.com/gl-matrix/-/gl-matrix-3.4.3.tgz", @@ -4219,7 +4425,7 @@ "version": "7.2.0", "resolved": "https://registry.npmmirror.com/glob/-/glob-7.2.0.tgz", "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", - "dev": true, + "devOptional": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -4378,6 +4584,12 @@ "node": ">=4" } }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "optional": true + }, "node_modules/history": { "version": "5.3.0", "resolved": "https://registry.npmmirror.com/history/-/history-5.3.0.tgz", @@ -4415,6 +4627,31 @@ "node": ">=8" } }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmmirror.com/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "dev": true, + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "optional": true, + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/husky": { "version": "8.0.3", "resolved": "https://registry.npmmirror.com/husky/-/husky-8.0.3.tgz", @@ -4513,7 +4750,7 @@ "version": "1.0.6", "resolved": "https://registry.npmmirror.com/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, + "devOptional": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -4523,7 +4760,7 @@ "version": "2.0.4", "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "devOptional": true }, "node_modules/ini": { "version": "1.3.8", @@ -4627,7 +4864,7 @@ "version": "3.0.0", "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -5100,7 +5337,7 @@ "version": "6.0.0", "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, + "devOptional": true, "dependencies": { "yallist": "^4.0.0" }, @@ -5108,6 +5345,30 @@ "node": ">=10" } }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "optional": true, + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "optional": true, + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/map-obj": { "version": "4.3.0", "resolved": "https://registry.npmmirror.com/map-obj/-/map-obj-4.3.0.tgz", @@ -5206,6 +5467,18 @@ "node": ">= 0.6" } }, + "node_modules/mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "optional": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/min-indent": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/min-indent/-/min-indent-1.0.1.tgz", @@ -5219,7 +5492,7 @@ "version": "3.1.2", "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, + "devOptional": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -5241,6 +5514,52 @@ "node": ">= 6" } }, + "node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "optional": true, + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "optional": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/mobx": { "version": "6.9.0", "resolved": "https://registry.npmmirror.com/mobx/-/mobx-6.9.0.tgz", @@ -5276,6 +5595,12 @@ "dev": true, "optional": true }, + "node_modules/nan": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", + "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", + "optional": true + }, "node_modules/nanoid": { "version": "3.3.6", "resolved": "https://registry.npmmirror.com/nanoid/-/nanoid-3.3.6.tgz", @@ -5347,11 +5672,46 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "peer": true }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "optional": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-releases": { "version": "2.0.12", "resolved": "https://registry.npmmirror.com/node-releases/-/node-releases-2.0.12.tgz", "integrity": "sha512-QzsYKWhXTWx8h1kIvqfnC++o0pEmpRQA/aenALsL2F4pqNVr7YzcdMlDij5WBnwftRbJCNJL/O7zdKaxKPHqgQ==" }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "optional": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/normalize-package-data": { "version": "3.0.3", "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", @@ -5367,21 +5727,6 @@ "node": ">=10" } }, - "node_modules/normalize-package-data/node_modules/semver": { - "version": "7.3.7", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", - "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-3.0.0.tgz", @@ -5400,6 +5745,27 @@ "node": ">=0.10.0" } }, + "node_modules/npmlog": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", + "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", + "optional": true, + "dependencies": { + "are-we-there-yet": "^2.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^3.0.0", + "set-blocking": "^2.0.0" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dev": true, + "dependencies": { + "boolbase": "^1.0.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz", @@ -5425,7 +5791,7 @@ "version": "1.4.0", "resolved": "https://registry.npmmirror.com/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, + "devOptional": true, "dependencies": { "wrappy": "1" } @@ -5535,6 +5901,25 @@ "node": ">= 0.10" } }, + "node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmmirror.com/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dev": true, + "dependencies": { + "entities": "^4.4.0" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dev": true, + "dependencies": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", @@ -5548,7 +5933,7 @@ "version": "1.0.1", "resolved": "https://registry.npmmirror.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -5585,6 +5970,27 @@ "node": ">=8" } }, + "node_modules/path2d-polyfill": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path2d-polyfill/-/path2d-polyfill-2.0.1.tgz", + "integrity": "sha512-ad/3bsalbbWhmBo0D6FZ4RNMwsLsPpL6gnvhuSaU5Vm7b06Kr5ubSltQQ0T7YKsiJQO+g22zJ4dJKNTXIyOXtA==", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/pdfjs-dist": { + "version": "4.0.269", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.0.269.tgz", + "integrity": "sha512-jjWO56tcOjnmPqDf8PmXDeZ781AGvpHMYI3HhNtaFKTRXXPaD1ArSrhVe38/XsrIQJ0onISCND/vuXaWJkiDWw==", + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "canvas": "^2.11.2", + "path2d-polyfill": "^2.0.1" + } + }, "node_modules/performance-now": { "version": "2.1.0", "resolved": "https://registry.npmmirror.com/performance-now/-/performance-now-2.1.0.tgz", @@ -7357,6 +7763,20 @@ "node": ">=8" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "optional": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz", @@ -7487,6 +7907,21 @@ "node": ">= 0.8.15" } }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "devOptional": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/rollup": { "version": "3.23.0", "resolved": "https://registry.npmmirror.com/rollup/-/rollup-3.23.0.tgz", @@ -7513,10 +7948,23 @@ } }, "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "peer": true + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] }, "node_modules/safer-buffer": { "version": "2.1.2", @@ -7561,6 +8009,21 @@ "compute-scroll-into-view": "^3.0.2" } }, + "node_modules/semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "devOptional": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/serialize-javascript": { "version": "6.0.0", "resolved": "https://registry.npmmirror.com/serialize-javascript/-/serialize-javascript-6.0.0.tgz", @@ -7570,6 +8033,12 @@ "randombytes": "^2.1.0" } }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "optional": true + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", @@ -7597,6 +8066,43 @@ "integrity": "sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw==", "dev": true }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "optional": true + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, + "node_modules/simple-get": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz", + "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", + "optional": true, + "dependencies": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmmirror.com/sisteransi/-/sisteransi-1.0.5.tgz", @@ -7754,6 +8260,15 @@ "node": ">=4" } }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "optional": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-convert": { "version": "0.2.1", "resolved": "https://registry.npmmirror.com/string-convert/-/string-convert-0.2.1.tgz", @@ -7763,7 +8278,7 @@ "version": "4.2.3", "resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, + "devOptional": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -7777,7 +8292,7 @@ "version": "6.0.1", "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, + "devOptional": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -8115,6 +8630,23 @@ "node": ">=6" } }, + "node_modules/tar": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz", + "integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==", + "optional": true, + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/terser": { "version": "5.12.1", "resolved": "https://registry.npmmirror.com/terser/-/terser-5.12.1.tgz", @@ -8221,6 +8753,12 @@ "resolved": "https://registry.npmmirror.com/toggle-selection/-/toggle-selection-1.0.6.tgz", "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==" }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "optional": true + }, "node_modules/trim-newlines": { "version": "3.0.1", "resolved": "https://registry.npmmirror.com/trim-newlines/-/trim-newlines-3.0.1.tgz", @@ -8399,7 +8937,7 @@ "version": "1.0.2", "resolved": "https://registry.npmmirror.com/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true + "devOptional": true }, "node_modules/uuid": { "version": "8.3.2", @@ -8499,6 +9037,12 @@ "node": ">=10.13.0" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "optional": true + }, "node_modules/webpack": { "version": "5.72.0", "resolved": "https://registry.npmmirror.com/webpack/-/webpack-5.72.0.tgz", @@ -8551,6 +9095,16 @@ "node": ">=10.13.0" } }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "optional": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", @@ -8566,6 +9120,15 @@ "node": ">= 8" } }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "optional": true, + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, "node_modules/word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmmirror.com/word-wrap/-/word-wrap-1.2.3.tgz", @@ -8579,7 +9142,7 @@ "version": "1.0.2", "resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "devOptional": true }, "node_modules/write-file-atomic": { "version": "5.0.1", @@ -8607,7 +9170,7 @@ "version": "4.0.0", "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "devOptional": true }, "node_modules/yaml": { "version": "1.10.2", @@ -9337,6 +9900,23 @@ "integrity": "sha512-QPcGmICAPbGLGb6F/yNf/KzKqvFx8z5qx3D1yFqVAjoFmXK35EgyW+cJ57Te3CNsmzblwtzakLGFqHPqrfb4Tw==", "requires": {} }, + "@mapbox/node-pre-gyp": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", + "integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==", + "optional": true, + "requires": { + "detect-libc": "^2.0.0", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.7", + "nopt": "^5.0.0", + "npmlog": "^5.0.1", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.11" + } + }, "@monaco-editor/loader": { "version": "1.4.0", "resolved": "https://registry.npmmirror.com/@monaco-editor/loader/-/loader-1.4.0.tgz", @@ -9783,17 +10363,6 @@ "natural-compare-lite": "^1.4.0", "semver": "^7.3.7", "tsutils": "^3.21.0" - }, - "dependencies": { - "semver": { - "version": "7.3.7", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", - "dev": true, - "requires": { - "lru-cache": "^6.0.0" - } - } } }, "@typescript-eslint/parser": { @@ -9849,17 +10418,6 @@ "is-glob": "^4.0.3", "semver": "^7.3.7", "tsutils": "^3.21.0" - }, - "dependencies": { - "semver": { - "version": "7.5.1", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.5.1.tgz", - "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", - "dev": true, - "requires": { - "lru-cache": "^6.0.0" - } - } } }, "@typescript-eslint/utils": { @@ -9876,17 +10434,6 @@ "@typescript-eslint/typescript-estree": "5.59.7", "eslint-scope": "^5.1.1", "semver": "^7.3.7" - }, - "dependencies": { - "semver": { - "version": "7.5.1", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.5.1.tgz", - "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", - "dev": true, - "requires": { - "lru-cache": "^6.0.0" - } - } } }, "@typescript-eslint/visitor-keys": { @@ -10137,6 +10684,12 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "peer": true }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "optional": true + }, "accessor-fn": { "version": "1.3.2", "resolved": "https://registry.npmmirror.com/accessor-fn/-/accessor-fn-1.3.2.tgz", @@ -10167,6 +10720,15 @@ "integrity": "sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA==", "dev": true }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "optional": true, + "requires": { + "debug": "4" + } + }, "ajv": { "version": "6.12.6", "resolved": "https://registry.npmmirror.com/ajv/-/ajv-6.12.6.tgz", @@ -10188,7 +10750,7 @@ "version": "5.0.1", "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true + "devOptional": true }, "ansi-styles": { "version": "3.2.1", @@ -10264,6 +10826,22 @@ "picomatch": "^2.0.4" } }, + "aproba": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", + "optional": true + }, + "are-we-there-yet": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", + "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", + "optional": true, + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + } + }, "argparse": { "version": "2.0.1", "resolved": "https://registry.npmmirror.com/argparse/-/argparse-2.0.1.tgz", @@ -10335,7 +10913,7 @@ "version": "1.0.2", "resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "devOptional": true }, "bezier-js": { "version": "4.1.1", @@ -10358,11 +10936,17 @@ "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", "dev": true }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, + "devOptional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -10422,6 +11006,17 @@ "resolved": "https://registry.npmmirror.com/caniuse-lite/-/caniuse-lite-1.0.30001489.tgz", "integrity": "sha512-x1mgZEXK8jHIfAxm+xgdpHpk50IN3z3q3zP261/WS+uvePxW8izXuCu6AHz0lkuYTlATDehiZ/tNyYBdSQsOUQ==" }, + "canvas": { + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz", + "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==", + "optional": true, + "requires": { + "@mapbox/node-pre-gyp": "^1.0.0", + "nan": "^2.17.0", + "simple-get": "^3.0.3" + } + }, "canvas-color-tracker": { "version": "1.1.5", "resolved": "https://registry.npmmirror.com/canvas-color-tracker/-/canvas-color-tracker-1.1.5.tgz", @@ -10454,6 +11049,35 @@ "supports-color": "^5.3.0" } }, + "cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmmirror.com/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dev": true, + "requires": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + } + }, + "cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dev": true, + "requires": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + } + }, "chokidar": { "version": "3.5.3", "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz", @@ -10470,6 +11094,12 @@ "readdirp": "~3.6.0" } }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "optional": true + }, "chrome-trace-event": { "version": "1.0.3", "resolved": "https://registry.npmmirror.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", @@ -10496,6 +11126,12 @@ "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, + "color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "optional": true + }, "colord": { "version": "2.9.3", "resolved": "https://registry.npmmirror.com/colord/-/colord-2.9.3.tgz", @@ -10517,7 +11153,13 @@ "version": "0.0.1", "resolved": "https://registry.npmmirror.com/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true + "devOptional": true + }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", + "optional": true }, "cookie": { "version": "0.3.1", @@ -10598,6 +11240,19 @@ "dev": true, "requires": {} }, + "css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dev": true, + "requires": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + } + }, "css-tree": { "version": "2.3.1", "resolved": "https://registry.npmmirror.com/css-tree/-/css-tree-2.3.1.tgz", @@ -10608,6 +11263,12 @@ "source-map-js": "^1.0.1" } }, + "css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "dev": true + }, "cssdb": { "version": "7.6.0", "resolved": "https://registry.npmmirror.com/cssdb/-/cssdb-7.6.0.tgz", @@ -10791,7 +11452,7 @@ "version": "4.3.4", "resolved": "https://registry.npmmirror.com/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, + "devOptional": true, "requires": { "ms": "2.1.2" }, @@ -10800,7 +11461,7 @@ "version": "2.1.2", "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "devOptional": true } } }, @@ -10833,6 +11494,15 @@ "resolved": "https://registry.npmmirror.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz", "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==" }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "optional": true, + "requires": { + "mimic-response": "^2.0.0" + } + }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmmirror.com/deep-is/-/deep-is-0.1.4.tgz", @@ -10851,6 +11521,18 @@ "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", "dev": true }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", + "optional": true + }, + "detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "optional": true + }, "detect-port-alt": { "version": "1.1.6", "resolved": "https://registry.npmmirror.com/detect-port-alt/-/detect-port-alt-1.1.6.tgz", @@ -10901,11 +11583,48 @@ "resolved": "https://registry.npmmirror.com/dom-align/-/dom-align-1.12.4.tgz", "integrity": "sha512-R8LUSEay/68zE5c8/3BDxiTEvgb4xZTF0RKmAHfiEVN3klfIpXfi2/QCoiWPccVQ0J/ZGdz9OjzL4uJEP/MRAw==" }, + "dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dev": true, + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + } + }, "dom-to-image": { "version": "2.6.0", "resolved": "https://registry.npmmirror.com/dom-to-image/-/dom-to-image-2.6.0.tgz", "integrity": "sha512-Dt0QdaHmLpjURjU7Tnu3AgYSF2LuOmksSGsUcE6ItvJoCWTBEmiMXcqBdNSAm9+QbbwD7JMoVsuuKX6ZVQv1qA==" }, + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true + }, + "domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmmirror.com/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dev": true, + "requires": { + "domelementtype": "^2.3.0" + } + }, + "domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dev": true, + "requires": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + } + }, "duplexer": { "version": "0.1.2", "resolved": "https://registry.npmmirror.com/duplexer/-/duplexer-0.1.2.tgz", @@ -10930,7 +11649,7 @@ "version": "8.0.0", "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "devOptional": true }, "emojis-list": { "version": "3.0.0", @@ -10947,6 +11666,12 @@ "tapable": "^2.2.0" } }, + "entities": { + "version": "4.5.0", + "resolved": "https://registry.npmmirror.com/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "dev": true + }, "errno": { "version": "0.1.8", "resolved": "https://registry.npmmirror.com/errno/-/errno-0.1.8.tgz", @@ -11453,17 +12178,6 @@ "requires": { "flatted": "^3.1.0", "rimraf": "^3.0.2" - }, - "dependencies": { - "rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "dev": true, - "requires": { - "glob": "^7.1.3" - } - } } }, "flatted": { @@ -11595,15 +12309,6 @@ "ajv-keywords": "^3.4.1" } }, - "semver": { - "version": "7.3.7", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", - "dev": true, - "requires": { - "lru-cache": "^6.0.0" - } - }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", @@ -11627,6 +12332,26 @@ "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==", "dev": true }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "optional": true, + "requires": { + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "optional": true, + "requires": { + "yallist": "^4.0.0" + } + } + } + }, "fs-monkey": { "version": "1.0.3", "resolved": "https://registry.npmmirror.com/fs-monkey/-/fs-monkey-1.0.3.tgz", @@ -11637,7 +12362,7 @@ "version": "1.0.0", "resolved": "https://registry.npmmirror.com/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "devOptional": true }, "fsevents": { "version": "2.3.2", @@ -11652,6 +12377,23 @@ "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true }, + "gauge": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "optional": true, + "requires": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.2", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.1", + "object-assign": "^4.1.1", + "signal-exit": "^3.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.2" + } + }, "gl-matrix": { "version": "3.4.3", "resolved": "https://registry.npmmirror.com/gl-matrix/-/gl-matrix-3.4.3.tgz", @@ -11661,7 +12403,7 @@ "version": "7.2.0", "resolved": "https://registry.npmmirror.com/glob/-/glob-7.2.0.tgz", "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", - "dev": true, + "devOptional": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -11792,6 +12534,12 @@ "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", "dev": true }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", + "optional": true + }, "history": { "version": "5.3.0", "resolved": "https://registry.npmmirror.com/history/-/history-5.3.0.tgz", @@ -11823,6 +12571,28 @@ "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==", "dev": true }, + "htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmmirror.com/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "dev": true, + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "optional": true, + "requires": { + "agent-base": "6", + "debug": "4" + } + }, "husky": { "version": "8.0.3", "resolved": "https://registry.npmmirror.com/husky/-/husky-8.0.3.tgz", @@ -11893,7 +12663,7 @@ "version": "1.0.6", "resolved": "https://registry.npmmirror.com/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, + "devOptional": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -11903,7 +12673,7 @@ "version": "2.0.4", "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "devOptional": true }, "ini": { "version": "1.3.8", @@ -11991,7 +12761,7 @@ "version": "3.0.0", "resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true + "devOptional": true }, "is-glob": { "version": "4.0.3", @@ -12373,11 +13143,28 @@ "version": "6.0.0", "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, + "devOptional": true, "requires": { "yallist": "^4.0.0" } }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "optional": true, + "requires": { + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "optional": true + } + } + }, "map-obj": { "version": "4.3.0", "resolved": "https://registry.npmmirror.com/map-obj/-/map-obj-4.3.0.tgz", @@ -12452,6 +13239,12 @@ "mime-db": "1.52.0" } }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "optional": true + }, "min-indent": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/min-indent/-/min-indent-1.0.1.tgz", @@ -12462,7 +13255,7 @@ "version": "3.1.2", "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, + "devOptional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -12478,6 +13271,39 @@ "kind-of": "^6.0.3" } }, + "minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "optional": true + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "optional": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "optional": true, + "requires": { + "yallist": "^4.0.0" + } + } + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "optional": true + }, "mobx": { "version": "6.9.0", "resolved": "https://registry.npmmirror.com/mobx/-/mobx-6.9.0.tgz", @@ -12502,6 +13328,12 @@ "dev": true, "optional": true }, + "nan": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", + "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", + "optional": true + }, "nanoid": { "version": "3.3.6", "resolved": "https://registry.npmmirror.com/nanoid/-/nanoid-3.3.6.tgz", @@ -12560,11 +13392,29 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "peer": true }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "optional": true, + "requires": { + "whatwg-url": "^5.0.0" + } + }, "node-releases": { "version": "2.0.12", "resolved": "https://registry.npmmirror.com/node-releases/-/node-releases-2.0.12.tgz", "integrity": "sha512-QzsYKWhXTWx8h1kIvqfnC++o0pEmpRQA/aenALsL2F4pqNVr7YzcdMlDij5WBnwftRbJCNJL/O7zdKaxKPHqgQ==" }, + "nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "optional": true, + "requires": { + "abbrev": "1" + } + }, "normalize-package-data": { "version": "3.0.3", "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", @@ -12575,17 +13425,6 @@ "is-core-module": "^2.5.0", "semver": "^7.3.4", "validate-npm-package-license": "^3.0.1" - }, - "dependencies": { - "semver": { - "version": "7.3.7", - "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", - "dev": true, - "requires": { - "lru-cache": "^6.0.0" - } - } } }, "normalize-path": { @@ -12600,6 +13439,27 @@ "integrity": "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==", "dev": true }, + "npmlog": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", + "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", + "optional": true, + "requires": { + "are-we-there-yet": "^2.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^3.0.0", + "set-blocking": "^2.0.0" + } + }, + "nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dev": true, + "requires": { + "boolbase": "^1.0.0" + } + }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz", @@ -12619,7 +13479,7 @@ "version": "1.4.0", "resolved": "https://registry.npmmirror.com/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, + "devOptional": true, "requires": { "wrappy": "1" } @@ -12705,6 +13565,25 @@ "integrity": "sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==", "dev": true }, + "parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmmirror.com/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dev": true, + "requires": { + "entities": "^4.4.0" + } + }, + "parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dev": true, + "requires": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + } + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", @@ -12715,7 +13594,7 @@ "version": "1.0.1", "resolved": "https://registry.npmmirror.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true + "devOptional": true }, "path-key": { "version": "3.1.1", @@ -12743,6 +13622,21 @@ "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true }, + "path2d-polyfill": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path2d-polyfill/-/path2d-polyfill-2.0.1.tgz", + "integrity": "sha512-ad/3bsalbbWhmBo0D6FZ4RNMwsLsPpL6gnvhuSaU5Vm7b06Kr5ubSltQQ0T7YKsiJQO+g22zJ4dJKNTXIyOXtA==", + "optional": true + }, + "pdfjs-dist": { + "version": "4.0.269", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.0.269.tgz", + "integrity": "sha512-jjWO56tcOjnmPqDf8PmXDeZ781AGvpHMYI3HhNtaFKTRXXPaD1ArSrhVe38/XsrIQJ0onISCND/vuXaWJkiDWw==", + "requires": { + "canvas": "^2.11.2", + "path2d-polyfill": "^2.0.1" + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmmirror.com/performance-now/-/performance-now-2.1.0.tgz", @@ -14040,6 +14934,17 @@ } } }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "optional": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, "readdirp": { "version": "3.6.0", "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz", @@ -14141,6 +15046,15 @@ "resolved": "https://registry.npmmirror.com/rgbcolor/-/rgbcolor-1.0.1.tgz", "integrity": "sha512-9aZLIrhRaD97sgVhtJOW6ckOEh6/GnvQtdVNfdZ6s67+3/XwLS9lBcQYzEEhYVeUowN7pRzMLsyGhK2i/xvWbw==" }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "devOptional": true, + "requires": { + "glob": "^7.1.3" + } + }, "rollup": { "version": "3.23.0", "resolved": "https://registry.npmmirror.com/rollup/-/rollup-3.23.0.tgz", @@ -14160,10 +15074,9 @@ } }, "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "peer": true + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "safer-buffer": { "version": "2.1.2", @@ -14205,6 +15118,15 @@ "compute-scroll-into-view": "^3.0.2" } }, + "semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "devOptional": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, "serialize-javascript": { "version": "6.0.0", "resolved": "https://registry.npmmirror.com/serialize-javascript/-/serialize-javascript-6.0.0.tgz", @@ -14214,6 +15136,12 @@ "randombytes": "^2.1.0" } }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", + "optional": true + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", @@ -14235,6 +15163,29 @@ "integrity": "sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw==", "dev": true }, + "signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "optional": true + }, + "simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "optional": true + }, + "simple-get": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz", + "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", + "optional": true, + "requires": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmmirror.com/sisteransi/-/sisteransi-1.0.5.tgz", @@ -14366,6 +15317,15 @@ "resolved": "https://registry.npmmirror.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==" }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "optional": true, + "requires": { + "safe-buffer": "~5.2.0" + } + }, "string-convert": { "version": "0.2.1", "resolved": "https://registry.npmmirror.com/string-convert/-/string-convert-0.2.1.tgz", @@ -14375,7 +15335,7 @@ "version": "4.2.3", "resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, + "devOptional": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -14386,7 +15346,7 @@ "version": "6.0.1", "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, + "devOptional": true, "requires": { "ansi-regex": "^5.0.1" } @@ -14670,6 +15630,20 @@ "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", "peer": true }, + "tar": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz", + "integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==", + "optional": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, "terser": { "version": "5.12.1", "resolved": "https://registry.npmmirror.com/terser/-/terser-5.12.1.tgz", @@ -14743,6 +15717,12 @@ "resolved": "https://registry.npmmirror.com/toggle-selection/-/toggle-selection-1.0.6.tgz", "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==" }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "optional": true + }, "trim-newlines": { "version": "3.0.1", "resolved": "https://registry.npmmirror.com/trim-newlines/-/trim-newlines-3.0.1.tgz", @@ -14863,7 +15843,7 @@ "version": "1.0.2", "resolved": "https://registry.npmmirror.com/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true + "devOptional": true }, "uuid": { "version": "8.3.2", @@ -14921,6 +15901,12 @@ "graceful-fs": "^4.1.2" } }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "optional": true + }, "webpack": { "version": "5.72.0", "resolved": "https://registry.npmmirror.com/webpack/-/webpack-5.72.0.tgz", @@ -14959,6 +15945,16 @@ "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", "peer": true }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "optional": true, + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", @@ -14968,6 +15964,15 @@ "isexe": "^2.0.0" } }, + "wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "optional": true, + "requires": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmmirror.com/word-wrap/-/word-wrap-1.2.3.tgz", @@ -14978,7 +15983,7 @@ "version": "1.0.2", "resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "devOptional": true }, "write-file-atomic": { "version": "5.0.1", @@ -15002,7 +16007,7 @@ "version": "4.0.0", "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "devOptional": true }, "yaml": { "version": "1.10.2", diff --git a/package.json b/package.json index 2fe9ba58..fce246da 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "lodash.debounce": "^4.0.8", "mobx": "^6.9.0", "mobx-react-lite": "^3.4.3", + "pdfjs-dist": "^4.0.269", "query-string": "^7.1.1", "react": "^18.2.0", "react-beautiful-dnd": "^13.1.1", @@ -73,6 +74,7 @@ "@typescript-eslint/parser": "^5.0.0", "@vitejs/plugin-react-swc": "^3.3.1", "autoprefixer": "^10.4.14", + "cheerio": "^1.0.0-rc.12", "ejs": "^3.1.9", "eslint": "^8.41.0", "eslint-config-prettier": "^8.8.0", @@ -82,7 +84,7 @@ "less": "^4.1.1", "postcss": "^8.4.12", "postcss-preset-env": "^8.4.1", - "prettier": "^2.3.2", + "prettier": "^2.8.8", "react-dev-utils": "^12.0.0", "stylelint": "^15.6.2", "stylelint-config-recommended": "^12.0.0", @@ -106,5 +108,6 @@ }, "ci": { "version": "8" - } + }, + "type": "module" } diff --git a/scripts/getDoc.js b/scripts/getDoc.js new file mode 100644 index 00000000..38c145f0 --- /dev/null +++ b/scripts/getDoc.js @@ -0,0 +1,88 @@ +import axios from 'axios'; +import fs from 'fs'; +import path from 'path' +import { fileURLToPath } from 'url'; +import * as cheerio from 'cheerio' + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +getDocIndex(); +async function getDocIndex(host = 'https://docs.nebula-graph.io') { + const data = await axios.get(host); + const version = data.data.match(/(([1-9]\d|[1-9])(\.([1-9]\d|\d)){2})/)?.[0]; + if (!version) { + return console.warn("no version found", data.data); + } + host = `${host}/${version}` + const homePageData = await axios.get(host); + const html = homePageData.data; + + const $ = cheerio.load(html); + const links = $('.md-nav--primary > .md-nav__list a'); + const linksMap = new Set(); + links.each((i, link) => { + const href = $(link).attr('href'); + if (href.includes('.pdf') > 0 || href.startsWith('#') || href.indexOf("1.nGQL-overview") > -1) return; + linksMap.add(href); + }); + + const length = linksMap.size; + const res = {}; + let now = 0; + // update or insert doc + for (let link of linksMap) { + const href = `${host}/${link}`; + let name = 'doc'; + try { + // if (link.indexOf('ngql-guide') >= 0 || link.indexOf('FAQ') >= 0) { + // name = 'ngql'; + // if (!res[name]) { + // res[name] = [] + // } + // console.log("get:", link) + // } else { + // console.log("skip:", link) + // continue; + // } + const data = await axios.get(href); + const html = data.data; + const $ = cheerio.load(html); + const main = $('article.md-content__inner'); + main.find('.headerlink').remove(); + main.find('.md-source-file').remove(); + main.find('.admonition').remove(); + const titleDom = main.find('> h1').first(); + titleDom.remove(); + const content = main.text(); + const title = titleDom.text(); + const finalContent = await makeDocTextForLLM(content); + console.log(finalContent); + const object = { + title: title, + content: finalContent, + url: href.replaceAll(host + '/3.ngql-guide/', ''), + type: 'doc' + } + console.log(`update:[${++now}/${length}]`, object.title); + res[name].push(object) + } catch (e) { + console.warn(e); + console.log(`update doc failed:[${++now}/${length}]`, href); + } + } + // save nowDocMap + console.log('saved') + for (let key in res) { + fs.writeFileSync(path.join(__dirname, `../app/utils/${key}.json`), JSON.stringify(res[key])); + } +} + +async function makeDocTextForLLM(content) { + //todo: make doc more short and clear with LLM + let contentArr = content.split('\n').filter(item => item.trim().length > 1); + contentArr = contentArr.filter(item => !/(^\+-+.*-+\+$)|(^\|.*\|$)/.test(item.replaceAll("\n", ''))); + const contentFinal = contentArr.join('\n').replaceAll('nebula>', ''); + return contentFinal; +} + +export default getDocIndex; \ No newline at end of file diff --git a/server/api/studio/etc/studio-api.yaml b/server/api/studio/etc/studio-api.yaml index 39a8bd20..3f14fd60 100644 --- a/server/api/studio/etc/studio-api.yaml +++ b/server/api/studio/etc/studio-api.yaml @@ -58,4 +58,8 @@ DB: # The maximum open connections of the pool. MaxOpenConns: 30 # The maximum idle connections of the pool. - MaxIdleConns: 10 \ No newline at end of file + MaxIdleConns: 10 +LLM: + GQLPath: "./data/llm" + GQLBatchSize: 1 + MaxBlockSize: 1 \ No newline at end of file diff --git a/server/api/studio/internal/config/config.go b/server/api/studio/internal/config/config.go index eee3c21c..654e3330 100644 --- a/server/api/studio/internal/config/config.go +++ b/server/api/studio/internal/config/config.go @@ -1,7 +1,6 @@ package config import ( - "io/ioutil" "os" "path/filepath" "reflect" @@ -64,6 +63,12 @@ type Config struct { MaxOpenConns int `json:",default=30"` MaxIdleConns int `json:",default=10"` } + + LLM struct { + GQLPath string `json:",default=./data/llm"` + GQLBatchSize int `json:",default=100"` + MaxBlockSize int `json:",default=0"` + } } type PathValidator struct { @@ -105,10 +110,13 @@ func (c *Config) Complete() { if _, err := os.Stat(filepath.Dir(abs)); os.IsNotExist(err) { os.MkdirAll(filepath.Dir(abs), os.ModePerm) } - ioutil.WriteFile(abs, []byte(""), os.ModePerm) + os.WriteFile(abs, []byte(""), os.ModePerm) } } } + if c.LLM.MaxBlockSize == 0 { + c.LLM.MaxBlockSize = 1024 * 1024 * 1024 + } } func (c *Config) InitConfig() error { diff --git a/server/api/studio/internal/handler/llm/addllmimportjobhandler.go b/server/api/studio/internal/handler/llm/addllmimportjobhandler.go new file mode 100644 index 00000000..2911a39f --- /dev/null +++ b/server/api/studio/internal/handler/llm/addllmimportjobhandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package llm + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func AddLLMImportJobHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.LLMImportRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := llm.NewAddLLMImportJobLogic(r.Context(), svcCtx) + data, err := l.AddLLMImportJob(req) + svcCtx.ResponseHandler.Handle(w, r, data, err) + } +} diff --git a/server/api/studio/internal/handler/llm/deletellmimportjobhandler.go b/server/api/studio/internal/handler/llm/deletellmimportjobhandler.go new file mode 100644 index 00000000..b62987b5 --- /dev/null +++ b/server/api/studio/internal/handler/llm/deletellmimportjobhandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package llm + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func DeleteLLMImportJobHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.DeleteLLMImportRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := llm.NewDeleteLLMImportJobLogic(r.Context(), svcCtx) + data, err := l.DeleteLLMImportJob(req) + svcCtx.ResponseHandler.Handle(w, r, data, err) + } +} diff --git a/server/api/studio/internal/handler/llm/downloadllmimportngqlhandler.go b/server/api/studio/internal/handler/llm/downloadllmimportngqlhandler.go new file mode 100644 index 00000000..7d140c52 --- /dev/null +++ b/server/api/studio/internal/handler/llm/downloadllmimportngqlhandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package llm + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func DownloadLLMImportNgqlHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.DownloadLLMImportNgqlRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := llm.NewDownloadLLMImportNgqlLogic(r.Context(), svcCtx) + data, err := l.DownloadLLMImportNgql(req) + svcCtx.ResponseHandler.Handle(w, r, data, err) + } +} diff --git a/server/api/studio/internal/handler/llm/getllmconfighandler.go b/server/api/studio/internal/handler/llm/getllmconfighandler.go new file mode 100644 index 00000000..88941641 --- /dev/null +++ b/server/api/studio/internal/handler/llm/getllmconfighandler.go @@ -0,0 +1,17 @@ +// Code generated by goctl. DO NOT EDIT. +package llm + +import ( + "net/http" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" +) + +func GetLLMConfigHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := llm.NewGetLLMConfigLogic(r.Context(), svcCtx) + data, err := l.GetLLMConfig() + svcCtx.ResponseHandler.Handle(w, r, data, err) + } +} diff --git a/server/api/studio/internal/handler/llm/getllmimportjobloghandler.go b/server/api/studio/internal/handler/llm/getllmimportjobloghandler.go new file mode 100644 index 00000000..7dd9a697 --- /dev/null +++ b/server/api/studio/internal/handler/llm/getllmimportjobloghandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package llm + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func GetLLMImportJobLogHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.LLMImportLogRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := llm.NewGetLLMImportJobLogLogic(r.Context(), svcCtx) + data, err := l.GetLLMImportJobLog(req) + svcCtx.ResponseHandler.Handle(w, r, data, err) + } +} diff --git a/server/api/studio/internal/handler/llm/getllmimportjobshandler.go b/server/api/studio/internal/handler/llm/getllmimportjobshandler.go new file mode 100644 index 00000000..81eee46b --- /dev/null +++ b/server/api/studio/internal/handler/llm/getllmimportjobshandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package llm + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func GetLLMImportJobsHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.LLMImportJobsRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := llm.NewGetLLMImportJobsLogic(r.Context(), svcCtx) + data, err := l.GetLLMImportJobs(req) + svcCtx.ResponseHandler.Handle(w, r, data, err) + } +} diff --git a/server/api/studio/internal/handler/llm/handlellmimportjobhandler.go b/server/api/studio/internal/handler/llm/handlellmimportjobhandler.go new file mode 100644 index 00000000..930feacb --- /dev/null +++ b/server/api/studio/internal/handler/llm/handlellmimportjobhandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package llm + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func HandleLLMImportJobHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.HandleLLMImportRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := llm.NewHandleLLMImportJobLogic(r.Context(), svcCtx) + data, err := l.HandleLLMImportJob(req) + svcCtx.ResponseHandler.Handle(w, r, data, err) + } +} diff --git a/server/api/studio/internal/handler/llm/llmconfighandler.go b/server/api/studio/internal/handler/llm/llmconfighandler.go new file mode 100644 index 00000000..77699a0d --- /dev/null +++ b/server/api/studio/internal/handler/llm/llmconfighandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package llm + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func LLMConfigHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.LLMConfigRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := llm.NewLLMConfigLogic(r.Context(), svcCtx) + err := l.LLMConfig(req) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + } +} diff --git a/server/api/studio/internal/handler/llm/llmproxyhandler.go b/server/api/studio/internal/handler/llm/llmproxyhandler.go new file mode 100644 index 00000000..5a91e80c --- /dev/null +++ b/server/api/studio/internal/handler/llm/llmproxyhandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package llm + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func LLMProxyHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.LLMRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithErrorMessage(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := llm.NewLLMProxyLogic(r.Context(), svcCtx) + data, err := l.LLMProxy(req) + svcCtx.ResponseHandler.Handle(w, r, data, err) + } +} diff --git a/server/api/studio/internal/handler/routes.go b/server/api/studio/internal/handler/routes.go index ad135454..021c1875 100644 --- a/server/api/studio/internal/handler/routes.go +++ b/server/api/studio/internal/handler/routes.go @@ -10,6 +10,7 @@ import ( gateway "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/handler/gateway" health "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/handler/health" importtask "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/handler/importtask" + llm "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/handler/llm" schema "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/handler/schema" sketches "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/handler/sketches" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" @@ -254,4 +255,54 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { }, }, ) + + server.AddRoutes( + []rest.Route{ + { + Method: http.MethodPost, + Path: "/api/llm", + Handler: llm.LLMProxyHandler(serverCtx), + }, + { + Method: http.MethodPost, + Path: "/api/config/llm", + Handler: llm.LLMConfigHandler(serverCtx), + }, + { + Method: http.MethodGet, + Path: "/api/config/llm", + Handler: llm.GetLLMConfigHandler(serverCtx), + }, + { + Method: http.MethodPost, + Path: "/api/llm/import/job", + Handler: llm.AddLLMImportJobHandler(serverCtx), + }, + { + Method: http.MethodGet, + Path: "/api/llm/import/jobs", + Handler: llm.GetLLMImportJobsHandler(serverCtx), + }, + { + Method: http.MethodGet, + Path: "/api/llm/import/job/log", + Handler: llm.GetLLMImportJobLogHandler(serverCtx), + }, + { + Method: http.MethodPost, + Path: "/api/llm/import/job/:action", + Handler: llm.HandleLLMImportJobHandler(serverCtx), + }, + { + Method: http.MethodDelete, + Path: "/api/llm/import/job/:jobId", + Handler: llm.DeleteLLMImportJobHandler(serverCtx), + }, + { + Method: http.MethodGet, + Path: "/api/llm/import/ngql", + Handler: llm.DownloadLLMImportNgqlHandler(serverCtx), + }, + }, + ) } diff --git a/server/api/studio/internal/logic/datasource/datasourceaddlogic.go b/server/api/studio/internal/logic/datasource/datasourceaddlogic.go index a45d861b..ca13510d 100644 --- a/server/api/studio/internal/logic/datasource/datasourceaddlogic.go +++ b/server/api/studio/internal/logic/datasource/datasourceaddlogic.go @@ -2,6 +2,7 @@ package datasource import ( "context" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" diff --git a/server/api/studio/internal/logic/datasource/datasourcelistcontentslogic.go b/server/api/studio/internal/logic/datasource/datasourcelistcontentslogic.go index fc9437ad..03cbe484 100644 --- a/server/api/studio/internal/logic/datasource/datasourcelistcontentslogic.go +++ b/server/api/studio/internal/logic/datasource/datasourcelistcontentslogic.go @@ -2,6 +2,7 @@ package datasource import ( "context" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" diff --git a/server/api/studio/internal/logic/datasource/datasourcepreviewfilelogic.go b/server/api/studio/internal/logic/datasource/datasourcepreviewfilelogic.go index becefcd5..c69609b3 100644 --- a/server/api/studio/internal/logic/datasource/datasourcepreviewfilelogic.go +++ b/server/api/studio/internal/logic/datasource/datasourcepreviewfilelogic.go @@ -2,6 +2,7 @@ package datasource import ( "context" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" diff --git a/server/api/studio/internal/logic/datasource/datasourceremovelogic.go b/server/api/studio/internal/logic/datasource/datasourceremovelogic.go index 36a6fea5..9dacbe8e 100644 --- a/server/api/studio/internal/logic/datasource/datasourceremovelogic.go +++ b/server/api/studio/internal/logic/datasource/datasourceremovelogic.go @@ -2,6 +2,7 @@ package datasource import ( "context" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" diff --git a/server/api/studio/internal/logic/datasource/datasourceupdatelogic.go b/server/api/studio/internal/logic/datasource/datasourceupdatelogic.go index 1d7208d9..626c334a 100644 --- a/server/api/studio/internal/logic/datasource/datasourceupdatelogic.go +++ b/server/api/studio/internal/logic/datasource/datasourceupdatelogic.go @@ -27,5 +27,4 @@ func NewDatasourceUpdateLogic(ctx context.Context, svcCtx *svc.ServiceContext) D func (l *DatasourceUpdateLogic) DatasourceUpdate(req types.DatasourceUpdateRequest) error { return service.NewDatasourceService(l.ctx, l.svcCtx).Update(req) - } diff --git a/server/api/studio/internal/logic/llm/addllmimportjoblogic.go b/server/api/studio/internal/logic/llm/addllmimportjoblogic.go new file mode 100644 index 00000000..9c1374ce --- /dev/null +++ b/server/api/studio/internal/logic/llm/addllmimportjoblogic.go @@ -0,0 +1,29 @@ +package llm + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type AddLLMImportJobLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewAddLLMImportJobLogic(ctx context.Context, svcCtx *svc.ServiceContext) AddLLMImportJobLogic { + return AddLLMImportJobLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *AddLLMImportJobLogic) AddLLMImportJob(req types.LLMImportRequest) (resp *types.LLMResponse, err error) { + return llm.NewLLMService(l.ctx, l.svcCtx).AddImportJob(&req) +} diff --git a/server/api/studio/internal/logic/llm/deletellmimportjoblogic.go b/server/api/studio/internal/logic/llm/deletellmimportjoblogic.go new file mode 100644 index 00000000..f3786c90 --- /dev/null +++ b/server/api/studio/internal/logic/llm/deletellmimportjoblogic.go @@ -0,0 +1,29 @@ +package llm + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type DeleteLLMImportJobLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewDeleteLLMImportJobLogic(ctx context.Context, svcCtx *svc.ServiceContext) DeleteLLMImportJobLogic { + return DeleteLLMImportJobLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *DeleteLLMImportJobLogic) DeleteLLMImportJob(req types.DeleteLLMImportRequest) (resp *types.LLMResponse, err error) { + return llm.NewLLMService(l.ctx, l.svcCtx).DeleteLLMImportJob(&req) +} diff --git a/server/api/studio/internal/logic/llm/downloadllmimportngqllogic.go b/server/api/studio/internal/logic/llm/downloadllmimportngqllogic.go new file mode 100644 index 00000000..0e6bbff5 --- /dev/null +++ b/server/api/studio/internal/logic/llm/downloadllmimportngqllogic.go @@ -0,0 +1,30 @@ +package llm + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type DownloadLLMImportNgqlLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewDownloadLLMImportNgqlLogic(ctx context.Context, svcCtx *svc.ServiceContext) DownloadLLMImportNgqlLogic { + return DownloadLLMImportNgqlLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *DownloadLLMImportNgqlLogic) DownloadLLMImportNgql(req types.DownloadLLMImportNgqlRequest) (resp *types.LLMResponse, err error) { + // todo: add your logic here and delete this line + + return +} diff --git a/server/api/studio/internal/logic/llm/getllmconfiglogic.go b/server/api/studio/internal/logic/llm/getllmconfiglogic.go new file mode 100644 index 00000000..82ad058b --- /dev/null +++ b/server/api/studio/internal/logic/llm/getllmconfiglogic.go @@ -0,0 +1,29 @@ +package llm + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type GetLLMConfigLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewGetLLMConfigLogic(ctx context.Context, svcCtx *svc.ServiceContext) GetLLMConfigLogic { + return GetLLMConfigLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *GetLLMConfigLogic) GetLLMConfig() (resp *types.LLMResponse, err error) { + return llm.NewLLMService(l.ctx, l.svcCtx).GetLLMConfig() +} diff --git a/server/api/studio/internal/logic/llm/getllmimportjobloglogic.go b/server/api/studio/internal/logic/llm/getllmimportjobloglogic.go new file mode 100644 index 00000000..ed411197 --- /dev/null +++ b/server/api/studio/internal/logic/llm/getllmimportjobloglogic.go @@ -0,0 +1,30 @@ +package llm + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type GetLLMImportJobLogLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewGetLLMImportJobLogLogic(ctx context.Context, svcCtx *svc.ServiceContext) GetLLMImportJobLogLogic { + return GetLLMImportJobLogLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *GetLLMImportJobLogLogic) GetLLMImportJobLog(req types.LLMImportLogRequest) (resp *types.LLMResponse, err error) { + // todo: add your logic here and delete this line + + return +} diff --git a/server/api/studio/internal/logic/llm/getllmimportjobslogic.go b/server/api/studio/internal/logic/llm/getllmimportjobslogic.go new file mode 100644 index 00000000..1c1eeb60 --- /dev/null +++ b/server/api/studio/internal/logic/llm/getllmimportjobslogic.go @@ -0,0 +1,29 @@ +package llm + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type GetLLMImportJobsLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewGetLLMImportJobsLogic(ctx context.Context, svcCtx *svc.ServiceContext) GetLLMImportJobsLogic { + return GetLLMImportJobsLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *GetLLMImportJobsLogic) GetLLMImportJobs(req types.LLMImportJobsRequest) (resp *types.LLMResponse, err error) { + return llm.NewLLMService(l.ctx, l.svcCtx).GetLLMImportJobs(&req) +} diff --git a/server/api/studio/internal/logic/llm/handlellmimportjoblogic.go b/server/api/studio/internal/logic/llm/handlellmimportjoblogic.go new file mode 100644 index 00000000..8f5cd303 --- /dev/null +++ b/server/api/studio/internal/logic/llm/handlellmimportjoblogic.go @@ -0,0 +1,29 @@ +package llm + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type HandleLLMImportJobLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewHandleLLMImportJobLogic(ctx context.Context, svcCtx *svc.ServiceContext) HandleLLMImportJobLogic { + return HandleLLMImportJobLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *HandleLLMImportJobLogic) HandleLLMImportJob(req types.HandleLLMImportRequest) (resp *types.LLMResponse, err error) { + return llm.NewLLMService(l.ctx, l.svcCtx).HandleLLMImportJob(&req) +} diff --git a/server/api/studio/internal/logic/llm/llmconfiglogic.go b/server/api/studio/internal/logic/llm/llmconfiglogic.go new file mode 100644 index 00000000..baa1af5c --- /dev/null +++ b/server/api/studio/internal/logic/llm/llmconfiglogic.go @@ -0,0 +1,29 @@ +package llm + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type LLMConfigLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewLLMConfigLogic(ctx context.Context, svcCtx *svc.ServiceContext) LLMConfigLogic { + return LLMConfigLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *LLMConfigLogic) LLMConfig(req types.LLMConfigRequest) error { + return llm.NewLLMService(l.ctx, l.svcCtx).LLMConfig(&req) +} diff --git a/server/api/studio/internal/logic/llm/llmproxylogic.go b/server/api/studio/internal/logic/llm/llmproxylogic.go new file mode 100644 index 00000000..03a5f63b --- /dev/null +++ b/server/api/studio/internal/logic/llm/llmproxylogic.go @@ -0,0 +1,29 @@ +package llm + +import ( + "context" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type LLMProxyLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewLLMProxyLogic(ctx context.Context, svcCtx *svc.ServiceContext) LLMProxyLogic { + return LLMProxyLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *LLMProxyLogic) LLMProxy(req types.LLMRequest) (resp *types.LLMResponse, err error) { + return llm.NewLLMService(l.ctx, l.svcCtx).LLMProxy(&req) +} diff --git a/server/api/studio/internal/model/db.go b/server/api/studio/internal/model/db.go index 79e4be82..591cb4e2 100644 --- a/server/api/studio/internal/model/db.go +++ b/server/api/studio/internal/model/db.go @@ -120,6 +120,8 @@ func InitDB(config *config.Config, db *gorm.DB) { &SchemaSnapshot{}, &Favorite{}, &File{}, + &LLMConfig{}, + &LLMJob{}, ) if err != nil { zap.L().Fatal(fmt.Sprintf("init taskInfo table fail: %s", err)) diff --git a/server/api/studio/internal/model/llm-job.go b/server/api/studio/internal/model/llm-job.go new file mode 100644 index 00000000..dc8a208b --- /dev/null +++ b/server/api/studio/internal/model/llm-job.go @@ -0,0 +1,49 @@ +package db + +import ( + "time" + + "gorm.io/datatypes" +) + +type LLMJob struct { + ID int `json:"id" gorm:"primaryKey;autoIncrement"` + UserName string `json:"user_name" gorm:"index:"` + Host string `json:"host" gorm:"index"` + JobID string `json:"job_id" gorm:"index:,unique"` + Space string `json:"space"` + SpaceSchemaString string `json:"space_schema_string"` + File string `json:"file"` + JobType string `json:"job_type"` + Status LLMStatus `json:"status"` + PromptTemplate string `json:"prompt_template"` + Process datatypes.JSON `json:"process"` + CreateTime time.Time `json:"create_time" gorm:"column:create_time;type:datetime;autoCreateTime"` + UpdateTime time.Time `json:"update_ime" gorm:"column:update_time;type:datetime;autoUpdateTime"` +} + +type Process struct { + TotalSize int `json:"total"` + CurrentSize int `json:"current"` + Ratio float64 `json:"ratio"` + FailedReason string `json:"failed_reason"` + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` +} + +type LLMStatus string + +const ( + LLMStatusRunning LLMStatus = "running" + LLMStatusSuccess LLMStatus = "success" + LLMStatusFailed LLMStatus = "failed" + LLMStatusCancel LLMStatus = "cancel" + LLMStatusPending LLMStatus = "pending" +) + +type LLMJobType string + +const ( + LLMJobTypeFile LLMJobType = "file" + LLMJobTypeFilePath LLMJobType = "file_path" +) diff --git a/server/api/studio/internal/model/llm.go b/server/api/studio/internal/model/llm.go new file mode 100644 index 00000000..e599f80b --- /dev/null +++ b/server/api/studio/internal/model/llm.go @@ -0,0 +1,21 @@ +package db + +type ModelVersion string + +const LLM3Dot5Turbo ModelVersion = "llm3.5-turbo" +const LLM4 ModelVersion = "llm4" + +type APIType string + +const OpenAI APIType = "openai" + +type LLMConfig struct { + ID int `json:"" gorm:"primaryKey;autoIncrement"` + URL string `json:"url"` + Key string `json:"key"` + APIType APIType `json:"apiType"` + ContextLengthLimit int `json:"contextLengthLimit"` + Config string `json:"config"` + Host string `json:"host"` + UserName string `json:"userName"` +} diff --git a/server/api/studio/internal/model/taskInfo.go b/server/api/studio/internal/model/taskInfo.go index 80ef395b..ec853bdd 100644 --- a/server/api/studio/internal/model/taskInfo.go +++ b/server/api/studio/internal/model/taskInfo.go @@ -21,6 +21,8 @@ type TaskInfo struct { ID int `gorm:"column:id;primaryKey;autoIncrement;"` BID string `gorm:"column:b_id;not null;type:char(32);uniqueIndex;comment:task id"` Address string `gorm:"column:address;type:varchar(255);"` + LLMJobID int `gorm:"column:llm_job_id;"` + LLMJob LLMJob `gorm:"foreignKey:LLMJobID;"` Name string `gorm:"column:name;type:varchar(255);"` Space string `gorm:"column:space;type:varchar(255);"` ImportAddress string `gorm:"column:import_address;"` diff --git a/server/api/studio/internal/model/vector.go b/server/api/studio/internal/model/vector.go new file mode 100644 index 00000000..cc72543d --- /dev/null +++ b/server/api/studio/internal/model/vector.go @@ -0,0 +1,9 @@ +package db + +type Vector struct { + ID int `json:"ID" gorm:"primaryKey;autoIncrement"` + Content string `json:"content"` + Title string `json:"title"` + Url string `json:"url"` + Vec string `json:"vec"` +} \ No newline at end of file diff --git a/server/api/studio/internal/service/favorite.go b/server/api/studio/internal/service/favorite.go index 8ffc61a6..b639cb66 100644 --- a/server/api/studio/internal/service/favorite.go +++ b/server/api/studio/internal/service/favorite.go @@ -66,6 +66,7 @@ func (s *favoriteService) Delete(request types.DeleteFavoriteRequest) error { } return nil } + func (s *favoriteService) DeleteAll() error { auth := s.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) host := auth.Address + ":" + strconv.Itoa(auth.Port) diff --git a/server/api/studio/internal/service/file.go b/server/api/studio/internal/service/file.go index c4cccb57..04346134 100644 --- a/server/api/studio/internal/service/file.go +++ b/server/api/studio/internal/service/file.go @@ -179,6 +179,7 @@ func (f *fileService) FileConfigUpdate(request types.FileConfigUpdateRequest) er } return nil } + func (f *fileService) FileUpload() error { dir := f.svcCtx.Config.File.UploadDir auth := f.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) @@ -203,9 +204,10 @@ func (f *fileService) FileUpload() error { return ecode.WithErrorMessage(ecode.ErrInternalServer, err, "upload failed") } for _, file := range files { - if file.Size == 0 { + if file.Size == 0 || file.Header.Get("Content-Type") != "text/csv" { continue } + //csv file charset check for importer charSet, err := checkCharset(file) if err != nil { logx.Infof("upload file error, check charset fail:%v", err) diff --git a/server/api/studio/internal/service/import.go b/server/api/studio/internal/service/import.go index 467783b5..10484a82 100644 --- a/server/api/studio/internal/service/import.go +++ b/server/api/studio/internal/service/import.go @@ -7,6 +7,7 @@ import ( "net/http" "path/filepath" "strconv" + "strings" "sync" db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" @@ -15,6 +16,7 @@ import ( "github.com/vesoft-inc/go-pkg/middleware" "github.com/vesoft-inc/nebula-importer/v4/pkg/config" configv3 "github.com/vesoft-inc/nebula-importer/v4/pkg/config/v3" + studioConfig "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/config" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service/importer" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" @@ -142,6 +144,7 @@ func (i *importService) updateDatasourceConfig(conf *types.CreateImportTaskReque } return &config, nil } + func updateConfig(conf config.Configurator, taskDir, uploadDir string) { confv3 := conf.(*configv3.Config) if confv3.Log == nil { @@ -331,6 +334,16 @@ func (i *importService) GetImportTaskLogNames(req *types.GetImportTaskLogNamesRe } func (i *importService) GetManyImportTaskLog(req *types.GetManyImportTaskLogRequest) (*types.GetManyImportTaskLogData, error) { + task := db.TaskInfo{ + BID: req.Id, + } + if err := db.CtxDB.Where(task).First(&task).Error; err != nil { + return nil, ecode.WithErrorMessage(ecode.ErrInternalDatabase, err) + } + if task.LLMJobID != 0 { + return i.GetAIImportLog(&task) + } + var taskEffect db.TaskEffect if err := db.CtxDB.Select("log").Where("task_id = ?", req.Id).First(&taskEffect).Error; err != nil { return nil, ecode.WithErrorMessage(ecode.ErrInternalDatabase, err) @@ -340,6 +353,22 @@ func (i *importService) GetManyImportTaskLog(req *types.GetManyImportTaskLogRequ return data, nil } +func (i *importService) GetAIImportLog(task *db.TaskInfo) (*types.GetManyImportTaskLogData, error) { + task.LLMJob.ID = task.LLMJobID + err := db.CtxDB.Where(task.LLMJob).First(&task.LLMJob).Error + if err != nil { + return nil, ecode.WithErrorMessage(ecode.ErrInternalDatabase, err) + } + jobPath := filepath.Join(studioConfig.GetConfig().LLM.GQLPath, fmt.Sprintf("%s/all.log", task.LLMJob.JobID)) + // read log + log, err := utils.ReadPartFile(jobPath) + if err != nil { + return nil, ecode.WithErrorMessage(ecode.ErrInternalServer, err) + } + data := &types.GetManyImportTaskLogData{Logs: strings.Join(log, "\n")} + return data, nil +} + func (i *importService) GetWorkingDir() (*types.GetWorkingDirResult, error) { return &types.GetWorkingDirResult{ TaskDir: i.svcCtx.Config.File.TasksDir, diff --git a/server/api/studio/internal/service/importer/importer.go b/server/api/studio/internal/service/importer/importer.go index 4c96efb4..94a1d332 100644 --- a/server/api/studio/internal/service/importer/importer.go +++ b/server/api/studio/internal/service/importer/importer.go @@ -6,8 +6,10 @@ import ( "regexp" "time" + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/llm" "github.com/zeromicro/go-zero/core/logx" ) @@ -156,6 +158,17 @@ func GetManyImportTask(address, username, space string, pageIndex, pageSize int) return nil, err } stats := t.Stats + var llmJob interface{} + if t.LLMJobID != 0 { + err = db.CtxDB.First(&t.LLMJob, t.LLMJobID).Error + if err == nil { + if runningJob := llm.GetRunningJob(t.LLMJob.JobID); runningJob != nil { + t.LLMJob.Process = runningJob.Process + } + llmJob = t.LLMJob + } + t.TaskStatus = string(t.LLMJob.Status) + } data := types.GetImportTaskData{ Id: t.BID, Status: t.TaskStatus, @@ -180,6 +193,7 @@ func GetManyImportTask(address, username, space string, pageIndex, pageSize int) FailedProcessed: stats.FailedProcessed, TotalProcessed: stats.TotalProcessed, }, + LLMJob: llmJob, } result.List = append(result.List, data) } diff --git a/server/api/studio/internal/service/llm/import.go b/server/api/studio/internal/service/llm/import.go new file mode 100644 index 00000000..161fd74d --- /dev/null +++ b/server/api/studio/internal/service/llm/import.go @@ -0,0 +1,149 @@ +package llm + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/vesoft-inc/go-pkg/response" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/config" + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/auth" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/llm" + "gorm.io/datatypes" +) + +func (g *llmService) AddImportJob(req *types.LLMImportRequest) (resp *types.LLMResponse, err error) { + auth := g.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) + config := db.LLMConfig{ + Host: fmt.Sprintf("%s:%d", auth.Address, auth.Port), + UserName: auth.Username, + } + err = db.CtxDB.Where(config).First(&config).Error + if err != nil { + return nil, err + } + space := req.Space + runes := []rune(space) + if len(runes) > 14 { + runes = runes[:14] + } + space = string(runes) + job := db.LLMJob{ + Space: req.Space, + File: req.File, + JobType: req.Type, + Status: db.LLMStatusPending, + Host: config.Host, + UserName: config.UserName, + PromptTemplate: req.PromptTemplate, + SpaceSchemaString: req.SpaceSchemaString, + JobID: space + "_" + time.Now().Format("20060102150405000"), + } + task := &db.TaskInfo{ + BID: job.JobID, + LLMJob: job, + Space: job.Space, + User: job.UserName, + Address: job.Host, + } + err = db.CtxDB.Create(task).Error + if err != nil { + return nil, err + } + return &types.LLMResponse{ + Data: response.StandardHandlerDataFieldAny(job), + }, nil +} + +func (g *llmService) GetLLMImportJobs(req *types.LLMImportJobsRequest) (resp *types.LLMResponse, err error) { + auth := g.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) + config := db.LLMConfig{ + Host: auth.Address, + UserName: auth.Username, + } + err = db.CtxDB.Where(config).First(&config).Error + if err != nil { + return nil, err + } + + var jobs []*db.LLMJob + err = db.CtxDB.Where("host = ? and user_name = ?", config.Host, config.UserName).Find(&jobs).Error + if err != nil { + return nil, err + } + // find from memory to avoid db update delay + for _, job := range jobs { + runningJob := llm.GetRunningJob(job.JobID) + if runningJob != nil { + job.Process = runningJob.Process + } + } + + return &types.LLMResponse{ + Data: response.StandardHandlerDataFieldAny(jobs), + }, nil +} + +func (g *llmService) HandleLLMImportJob(req *types.HandleLLMImportRequest) (resp *types.LLMResponse, err error) { + var job db.LLMJob + err = db.CtxDB.Where("job_id = ?", req.JobID).First(&job).Error + if err != nil { + return nil, err + } + + if req.Action == "cancel" { + job.Status = db.LLMStatusCancel + llm.ChangeRunningJobStatus(job.JobID, db.LLMStatusCancel) + } + if req.Action == "rerun" { + job.Status = db.LLMStatusPending + // datatypes.JSON + job.Process = datatypes.JSON("{}") + //delete log & ngql + jobPath := filepath.Join(config.GetConfig().LLM.GQLPath, job.JobID) + err = os.RemoveAll(jobPath) + if err != nil { + return nil, fmt.Errorf("remove job path error: %v", err) + } + } + + err = db.CtxDB.Save(&job).Error + if err != nil { + return nil, err + } + + return &types.LLMResponse{ + Data: response.StandardHandlerDataFieldAny(job), + }, nil +} + +func (g *llmService) DeleteLLMImportJob(req *types.DeleteLLMImportRequest) (resp *types.LLMResponse, err error) { + var job db.LLMJob + err = db.CtxDB.Where("job_id = ?", req.JobID).First(&job).Error + if err != nil { + return nil, fmt.Errorf("get job error: %v", err) + } + + err = db.CtxDB.Delete(&job).Error + if err != nil { + return nil, fmt.Errorf("delete job error: %v", err) + } + //delete task info + err = db.CtxDB.Where("llm_job_id = ?", job.ID).Delete(&db.TaskInfo{}).Error + if err != nil { + return nil, err + } + //delete log & ngql + jobPath := filepath.Join(config.GetConfig().LLM.GQLPath, job.JobID) + err = os.RemoveAll(jobPath) + if err != nil { + return nil, fmt.Errorf("remove job path error: %v", err) + } + + return &types.LLMResponse{ + Data: response.StandardHandlerDataFieldAny(job), + }, nil +} diff --git a/server/api/studio/internal/service/llm/llm.go b/server/api/studio/internal/service/llm/llm.go new file mode 100644 index 00000000..633f483c --- /dev/null +++ b/server/api/studio/internal/service/llm/llm.go @@ -0,0 +1,99 @@ +package llm + +import ( + "context" + "fmt" + + "github.com/vesoft-inc/go-pkg/response" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/config" + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/auth" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/llm" + "github.com/zeromicro/go-zero/core/logx" +) + +type LLMService interface { + LLMProxy(req *types.LLMRequest) (resp *types.LLMResponse, err error) + LLMConfig(req *types.LLMConfigRequest) error + GetLLMConfig() (resp *types.LLMResponse, err error) + AddImportJob(req *types.LLMImportRequest) (resp *types.LLMResponse, err error) + GetLLMImportJobs(req *types.LLMImportJobsRequest) (resp *types.LLMResponse, err error) + HandleLLMImportJob(req *types.HandleLLMImportRequest) (resp *types.LLMResponse, err error) + DeleteLLMImportJob(req *types.DeleteLLMImportRequest) (resp *types.LLMResponse, err error) +} + +type llmService struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewLLMService(ctx context.Context, svcCtx *svc.ServiceContext) LLMService { + return &llmService{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +// LLMProxy: proxy llm request to llm server only stream false +// if you need stream ,you can use websocket +func (g *llmService) LLMProxy(req *types.LLMRequest) (resp *types.LLMResponse, err error) { + auth := g.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) + data, err := llm.Fetch(auth, req.Data, func(str string) {}) + if err != nil { + return nil, err + } + resp = &types.LLMResponse{ + Data: data, + } + return resp, nil +} + +func (g *llmService) GetLLMConfig() (resp *types.LLMResponse, err error) { + auth := g.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) + llmConfig := db.LLMConfig{ + Host: fmt.Sprintf("%s:%d", auth.Address, auth.Port), + UserName: auth.Username, + } + res := db.CtxDB.Where(llmConfig).First(&llmConfig) + if res.RowsAffected == 0 { + return nil, nil + } + return &types.LLMResponse{ + Data: response.StandardHandlerDataFieldAny(map[string]any{ + "gqlPath": config.GetConfig().LLM.GQLPath, + "config": llmConfig, + }), + }, nil +} + +func (g *llmService) LLMConfig(req *types.LLMConfigRequest) (err error) { + auth := g.ctx.Value(auth.CtxKeyUserInfo{}).(*auth.AuthData) + oldConfig := db.LLMConfig{ + Host: auth.Address, + UserName: auth.Username, + } + if db.CtxDB.Find(&oldConfig).RowsAffected > 0 { + res := db.CtxDB.Where(oldConfig).Delete(&db.LLMConfig{}, oldConfig.ID) + if res.Error != nil { + return res.Error + } + } + llmConfig := db.LLMConfig{ + URL: req.URL, + Key: req.Key, + APIType: db.APIType(req.APIType), + Config: req.Config, + Host: fmt.Sprintf("%s:%d", auth.Address, auth.Port), + UserName: auth.Username, + ContextLengthLimit: req.MaxContextLength, + } + res := db.CtxDB.Create(&llmConfig) + if res.Error != nil { + return res.Error + } + return nil +} diff --git a/server/api/studio/internal/types/types.go b/server/api/studio/internal/types/types.go index a4b09799..eff5011f 100644 --- a/server/api/studio/internal/types/types.go +++ b/server/api/studio/internal/types/types.go @@ -219,6 +219,7 @@ type GetImportTaskData struct { UpdateTime int64 `json:"updateTime"` Stats ImportTaskStats `json:"stats"` RawConfig string `json:"rawConfig"` + LLMJob interface{} `json:"llmJob"` } type ImportTaskStats struct { @@ -465,3 +466,51 @@ type DatasourcePreviewFileRequest struct { type DatasourcePreviewFileData struct { Contents []string `json:"contents"` } + +type LLMRequest struct { + Data map[string]interface{} `json:"data"` +} + +type LLMResponse struct { + Data interface{} `json:"data"` +} + +type LLMConfigRequest struct { + URL string `json:"url"` + Key string `json:"key,optional"` + APIType string `json:"apiType"` + MaxContextLength int `json:"maxContextLength"` + Config string `json:"config,optional"` +} + +type LLMImportRequest struct { + Space string `json:"space"` + File string `json:"file,optional"` + FilePath string `json:"filePath,optional"` + Type string `json:"type"` + PromptTemplate string `json:"promptTemplate"` + SpaceSchemaString string `json:"spaceSchemaString"` +} + +type LLMImportJobsRequest struct { + Page int `json:"page"` + PageSize int `json:"pageSize"` + Space string `json:"space,optional"` +} + +type LLMImportLogRequest struct { + JobID string `json:"jobId"` +} + +type HandleLLMImportRequest struct { + JobID string `json:"jobId"` + Action string `path:"action"` +} + +type DeleteLLMImportRequest struct { + JobID string `path:"jobId"` +} + +type DownloadLLMImportNgqlRequest struct { + JobID string `json:"jobId"` +} diff --git a/server/api/studio/pkg/auth/authorize.go b/server/api/studio/pkg/auth/authorize.go index 4460e181..7bd83285 100644 --- a/server/api/studio/pkg/auth/authorize.go +++ b/server/api/studio/pkg/auth/authorize.go @@ -38,6 +38,8 @@ type ( } ) +var CtxUserInfoMap map[string]AuthData = make(map[string]AuthData) + // set the timeout for the graph service: 8 hours // once the timeout is reached, the connection will be closed // all requests running ngql will be failed, so keepping a long timeout is necessary, make the connection alive @@ -135,6 +137,14 @@ func ParseConnectDBParams(params *types.ConnectDBParams, config *config.Config) if err != nil { return "", err } + // cache auth info key for llm import use + key := fmt.Sprintf("%s:%d:%s", params.Address, params.Port, username) + CtxUserInfoMap[key] = AuthData{ + Address: params.Address, + Port: params.Port, + Username: username, + Password: password, + } tokenString, err := CreateToken( &AuthData{ @@ -146,7 +156,7 @@ func ParseConnectDBParams(params *types.ConnectDBParams, config *config.Config) }, config, ) - return tokenString, nil + return tokenString, err } func AuthMiddlewareWithCtx(svcCtx *svc.ServiceContext) rest.Middleware { diff --git a/server/api/studio/pkg/client/client.go b/server/api/studio/pkg/client/client.go index 5dbec3d0..03033fb6 100644 --- a/server/api/studio/pkg/client/client.go +++ b/server/api/studio/pkg/client/client.go @@ -62,9 +62,7 @@ type ClientInfo struct { ClientID string } -var ( - clientPool = utils.NewMutexMap[*Client]() -) +var clientPool = utils.NewMutexMap[*Client]() var log = newNebulaLogger() @@ -111,7 +109,6 @@ func NewClient(address string, port int, username string, password string, conf } session, err := client.getSession() - if err != nil { CloseClient(nsid) return nil, err diff --git a/server/api/studio/pkg/client/clientCommand.go b/server/api/studio/pkg/client/clientCommand.go index 3c8b2d24..efd9bcb2 100644 --- a/server/api/studio/pkg/client/clientCommand.go +++ b/server/api/studio/pkg/client/clientCommand.go @@ -9,8 +9,10 @@ import ( "time" ) -type ParameterList []string -type ParameterMap map[string]interface{} +type ( + ParameterList []string + ParameterMap map[string]interface{} +) // Console side commands const ( diff --git a/server/api/studio/pkg/client/pool.go b/server/api/studio/pkg/client/pool.go index 9c0789a4..1a75b387 100644 --- a/server/api/studio/pkg/client/pool.go +++ b/server/api/studio/pkg/client/pool.go @@ -76,7 +76,7 @@ func getValue(valWarp *nebula.ValueWrapper) (Any, error) { } func getBasicValue(valWarp *nebula.ValueWrapper) (Any, error) { - var valType = valWarp.GetType() + valType := valWarp.GetType() if valType == "null" { value, err := valWarp.AsNull() switch value { @@ -121,6 +121,7 @@ func getBasicValue(valWarp *nebula.ValueWrapper) (Any, error) { } return "", nil } + func getID(idWarp nebula.ValueWrapper) Any { idType := idWarp.GetType() var vid Any @@ -230,7 +231,7 @@ func getListInfo(valWarp *nebula.ValueWrapper, listType string, _verticesParsedL return err } for _, v := range valueList { - var props = make(map[string]Any) + props := make(map[string]Any) vType := v.GetType() props["type"] = vType if vType == "vertex" { @@ -496,7 +497,7 @@ func parseExecuteData(response SingleResponse) (ParsedResult, error) { result.Headers = []string{"id", "name", "dependencies", "profiling data", "operator info"} rows := res.MakePlanByRow() for i := 0; i < len(rows); i++ { - var rowValue = make(map[string]Any) + rowValue := make(map[string]Any) rowValue["id"] = rows[i][0] rowValue["name"] = rows[i][1] rowValue["dependencies"] = rows[i][2] @@ -506,7 +507,7 @@ func parseExecuteData(response SingleResponse) (ParsedResult, error) { } return result, nil } else { - var rowValue = make(map[string]Any) + rowValue := make(map[string]Any) result.Headers = append(result.Headers, "format") if format == "dot" { rowValue["format"] = res.MakeDotGraph() @@ -525,10 +526,10 @@ func parseExecuteData(response SingleResponse) (ParsedResult, error) { result.Headers = colNames for i := 0; i < rowSize; i++ { - var rowValue = make(map[string]Any) - var _verticesParsedList = make(list, 0) - var _edgesParsedList = make(list, 0) - var _pathsParsedList = make(list, 0) + rowValue := make(map[string]Any) + _verticesParsedList := make(list, 0) + _edgesParsedList := make(list, 0) + _pathsParsedList := make(list, 0) for j := 0; j < colSize; j++ { record, err := res.GetRowValuesByIndex(i) @@ -546,17 +547,17 @@ func parseExecuteData(response SingleResponse) (ParsedResult, error) { rowValue[result.Headers[j]] = value valueType := rowData.GetType() if valueType == "vertex" { - var parseValue = make(map[string]Any) + parseValue := make(map[string]Any) parseValue, err = getVertexInfo(rowData, parseValue) parseValue["type"] = "vertex" _verticesParsedList = append(_verticesParsedList, parseValue) } else if valueType == "edge" { - var parseValue = make(map[string]Any) + parseValue := make(map[string]Any) parseValue, err = getEdgeInfo(rowData, parseValue) parseValue["type"] = "edge" _edgesParsedList = append(_edgesParsedList, parseValue) } else if valueType == "path" { - var parseValue = make(map[string]Any) + parseValue := make(map[string]Any) parseValue, err = getPathInfo(rowData, parseValue) parseValue["type"] = "path" _pathsParsedList = append(_pathsParsedList, parseValue) diff --git a/server/api/studio/pkg/filestore/sftpstore_test.go b/server/api/studio/pkg/filestore/sftpstore_test.go index ff790016..3800a5ce 100644 --- a/server/api/studio/pkg/filestore/sftpstore_test.go +++ b/server/api/studio/pkg/filestore/sftpstore_test.go @@ -1,10 +1,11 @@ package filestore import ( - "github.com/agiledragon/gomonkey/v2" "reflect" "testing" + "github.com/agiledragon/gomonkey/v2" + "github.com/pkg/sftp" ) diff --git a/server/api/studio/pkg/llm/.gitignore b/server/api/studio/pkg/llm/.gitignore new file mode 100644 index 00000000..6320cd24 --- /dev/null +++ b/server/api/studio/pkg/llm/.gitignore @@ -0,0 +1 @@ +data \ No newline at end of file diff --git a/server/api/studio/pkg/llm/fetch.go b/server/api/studio/pkg/llm/fetch.go new file mode 100644 index 00000000..6c9de871 --- /dev/null +++ b/server/api/studio/pkg/llm/fetch.go @@ -0,0 +1,58 @@ +package llm + +import ( + "fmt" + "net/http" + "time" + + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/auth" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/llm/transformer" +) + +func Fetch(auth *auth.AuthData, req map[string]any, callback func(str string)) (map[string]any, error) { + var config = db.LLMConfig{ + Host: auth.Address, + UserName: auth.Username, + } + err := db.CtxDB.First(&config).Error + if err != nil { + return nil, err + } + return FetchWithLLMConfig(&config, req, callback) +} + +func FetchWithLLMConfig(config *db.LLMConfig, req map[string]any, callback func(str string)) (map[string]any, error) { + + // Convert the request parameters to a JSON string + var transform transformer.Handler + if config.APIType == "openai" { + transform = &transformer.OpenAI{} + } else if config.APIType == "qwen" { + transform = &transformer.Qwen{} + } + httpReq, err := transform.HandleRequest(req, config) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP request: %v", err) + } + // Send the HTTP request + client := &http.Client{ + Timeout: 60 * time.Second, + } + resp, err := client.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + defer client.CloseIdleConnections() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to send HTTP request: %v,%v", resp.StatusCode, resp.Body) + } + return transform.HandleResponse(resp, callback) +} + +type Document struct { + Title string `json:"title"` + Content string `json:"content"` + Url string `json:"url"` +} diff --git a/server/api/studio/pkg/llm/importjob.go b/server/api/studio/pkg/llm/importjob.go new file mode 100644 index 00000000..50dc03d2 --- /dev/null +++ b/server/api/studio/pkg/llm/importjob.go @@ -0,0 +1,538 @@ +package llm + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + nebula_go "github.com/vesoft-inc/nebula-go/v3" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/config" + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/auth" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/client" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/pdf" + "gorm.io/datatypes" +) + +type Field struct { + Name string `json:"name"` + DataType string `json:"dataType"` + Nullable bool `json:"nullable"` +} +type NodeType struct { + Type string `json:"type"` + Props []Field `json:"props"` +} +type EdgeType struct { + Type string `json:"type"` + Props []Field `json:"props"` +} +type Schema struct { + Space string `json:"spaceName"` + VidType string `json:"vidType"` + NodeTypes []NodeType `json:"nodeTypes"` + EdgeTypes []EdgeType `json:"edgeTypes"` +} +type ImportJob struct { + CacheNodes map[string]Node + CacheEdges map[string]map[string]Edge + Prompt string + LLMJob *db.LLMJob + LLMConfig *db.LLMConfig + AuthData *auth.AuthData + NSID string + Process *db.Process + logFile *os.File + Schema Schema + SchemaMap map[string]map[string]Field +} + +func RunFileJob(job *db.LLMJob) { + llmJob := ImportJob{ + CacheNodes: make(map[string]Node), + CacheEdges: make(map[string]map[string]Edge), + LLMJob: job, + Process: &db.Process{ + TotalSize: 0, + CurrentSize: 0, + Ratio: 0, + PromptTokens: 0, + CompletionTokens: 0, + }, + } + err := llmJob.AddLogFile() + if err != nil { + llmJob.SetJobFailed(err) + return + } + defer llmJob.CloseLogFile() + go llmJob.SyncProcess() + defer func() { + if err := recover(); err != nil { + llmJob.WriteLogFile(fmt.Sprintf("panic: %v", err), "error") + llmJob.SetJobFailed(err) + } + processJson, err := json.Marshal(llmJob.Process) + if err != nil { + llmJob.WriteLogFile(fmt.Sprintf("marshal process error: %v", err), "error") + } + err = db.CtxDB.Model(&job).Where("job_id = ?", job.JobID).Updates(map[string]interface{}{ + "process": datatypes.JSON(processJson), + "status": job.Status, + }).Error + if err != nil { + llmJob.WriteLogFile(fmt.Sprintf("update process error: %v", err), "error") + return + } + }() + + llmJob.Process.Ratio = 0.01 + err = llmJob.ParseSchema(job.SpaceSchemaString) + if err != nil { + llmJob.WriteLogFile(fmt.Sprintf("parse schema error: %v", err), "error") + llmJob.SetJobFailed(err) + return + } + + llmConfig := db.LLMConfig{ + Host: job.Host, + UserName: job.UserName, + } + err = db.CtxDB.Where(llmConfig).First(&llmConfig).Error + if err != nil { + llmJob.WriteLogFile(fmt.Sprintf("get llm config error: %v", err), "error") + llmJob.SetJobFailed(err) + return + } + llmJob.LLMConfig = &llmConfig + llmJob.Process.Ratio = 0.03 + + connectInfo, ok := auth.CtxUserInfoMap[fmt.Sprintf("%s:%s", job.Host, job.UserName)] + if !ok { + llmJob.WriteLogFile(fmt.Sprintf("get connect info error: %v", err), "error") + llmJob.SetJobFailed(err) + return + } + llmJob.AuthData = &connectInfo + clientInfo, err := client.NewClient(connectInfo.Address, connectInfo.Port, connectInfo.Username, connectInfo.Password, nebula_go.GetDefaultConf()) + if err != nil { + llmJob.WriteLogFile(fmt.Sprintf("create client error: %v", err), "error") + return + } + llmJob.NSID = clientInfo.ClientID + llmJob.Process.Ratio = 0.05 + + llmJob.WriteLogFile(fmt.Sprintf("start run file job, file path: %s", job.File), "info") + + filePath := path.Join(config.GetConfig().File.UploadDir, llmJob.LLMJob.File) + text, err := llmJob.ReadFile(filePath) + if err != nil { + llmJob.WriteLogFile(fmt.Sprintf("read file error: %v", err), "error") + llmJob.SetJobFailed(err) + return + } + blocks, err := llmJob.SplitText(text) + if err != nil { + llmJob.WriteLogFile(fmt.Sprintf("split text error: %v", err), "error") + llmJob.SetJobFailed(err) + return + } + llmJob.Process.Ratio = 0.07 + err = llmJob.QueryBlocks(blocks) + if err != nil { + llmJob.WriteLogFile(fmt.Sprintf("query blocks error: %v", err), "error") + llmJob.SetJobFailed(err) + return + } + llmJob.Process.Ratio = 0.8 + gqls, err := llmJob.MakeGQLFile() + if err != nil { + llmJob.WriteLogFile(fmt.Sprintf("make gql file error: %v", err), "error") + llmJob.SetJobFailed(err) + return + } + if IsRunningJobStopped(job.JobID) { + return + } + llmJob.Process.Ratio = 0.9 + + llmJob.RunGQLFile(gqls) + llmJob.Process.Ratio = 1 + llmJob.LLMJob.Status = db.LLMStatusSuccess +} + +func (i *ImportJob) SyncProcess() { + for { + // stop + if i.LLMJob.Status != db.LLMStatusRunning { + return + } + jsonStr, err := json.Marshal(i.Process) + if err != nil { + i.WriteLogFile(fmt.Sprintf("marshal process error: %v", err), "error") + continue + } + i.LLMJob.Process = datatypes.JSON(jsonStr) + time.Sleep(time.Second) + } +} + +func (i *ImportJob) SetJobFailed(failedErr any) { + i.LLMJob.Status = db.LLMStatusFailed + i.Process.FailedReason = fmt.Sprintf("%v", failedErr) +} + +func (i *ImportJob) AddLogFile() error { + // mkdir + err := os.MkdirAll(filepath.Join(config.GetConfig().LLM.GQLPath, i.LLMJob.JobID), 0755) + if err != nil { + return err + } + logFile := filepath.Join(config.GetConfig().LLM.GQLPath, fmt.Sprintf("%s/all.log", i.LLMJob.JobID)) + file, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + i.logFile = file + i.WriteLogFile(fmt.Sprintf("add log file success, file path: %s", logFile), "info") + i.WriteLogFile(fmt.Sprintf("now task info: \n%v\n config:%v", i.LLMJob, i.LLMConfig), "info") + return nil +} + +func (i *ImportJob) CloseLogFile() { + if i.logFile != nil { + i.logFile.Close() + } +} + +func (i *ImportJob) WriteLogFile(str string, typ string) { + if i.logFile != nil { + i.logFile.WriteString(fmt.Sprintf("%s[%s] %s\n", time.Now().Format("2006-01-02 15:04:05"), typ, str)) + } +} + +func (i *ImportJob) ReadFile(filePath string) (string, error) { + if filePath == "" { + return "", fmt.Errorf("file path is empty") + } + + file, err := os.Open(filePath) + if err != nil { + return "", fmt.Errorf("open file error: %v", err) + } + stat, err := file.Stat() + if err != nil { + return "", fmt.Errorf("get file stat error: %v", err) + } + size := stat.Size() + i.Process.TotalSize = int(size) + + if strings.HasSuffix(filePath, ".pdf") { + file.Close() + return pdf.ReadPDFFile(filePath) + } + defer file.Close() + bytes, err := io.ReadAll(file) + if err != nil { + return "", fmt.Errorf("read file error: %v", err) + } + i.WriteLogFile(fmt.Sprintf("read file success, file path: %s", filePath), "info") + return string(bytes), nil +} + +func (i *ImportJob) SplitText(str string) (blocks []string, err error) { + // split text to each blocks for llm context length limit + blocks = make([]string, 0) + lines := strings.Split(str, "\n") + block := "" + for _, line := range lines { + if line == "\n" { + continue + } + if len(block)+len(line)+len(i.Prompt) > i.LLMConfig.ContextLengthLimit { + blocks = append(blocks, block) + block = "" + } + block += line + "\n" + } + if block != "" { + blocks = append(blocks, block) + } + i.WriteLogFile(fmt.Sprintf("split text success, blocks length: %d", len(blocks)), "info") + + return blocks, nil +} + +func (i *ImportJob) ParseSchema(text string) error { + schema := Schema{} + err := json.Unmarshal([]byte(text), &schema) + if err != nil { + return err + } + i.Schema = schema + i.SchemaMap = make(map[string]map[string]Field) + for _, tag := range schema.NodeTypes { + i.SchemaMap[tag.Type] = make(map[string]Field) + for _, field := range tag.Props { + i.SchemaMap[tag.Type][field.Name] = field + } + } + for _, edge := range schema.EdgeTypes { + i.SchemaMap[edge.Type] = make(map[string]Field) + for _, field := range edge.Props { + i.SchemaMap[edge.Type][field.Name] = field + } + } + return nil +} + +type LLMResult struct { + Nodes []Node `json:"nodes"` + Edges []Edge `json:"edges"` +} +type Node struct { + Name string `json:"name"` + Type string `json:"type"` + Props map[string]any `json:"props"` +} + +type Edge struct { + Src string `json:"src"` + Dst string `json:"dst"` + EdgeType string `json:"edgeType"` + Props map[string]any `json:"props"` +} + +func (i *ImportJob) ParseText(text string) { + jsonObj := LLMResult{} + err := json.Unmarshal([]byte(text), &jsonObj) + if err != nil { + i.WriteLogFile(fmt.Sprintf("parse text error: %v, str:%s", err, text), "error") + return + } + for _, node := range jsonObj.Nodes { + nowNode, ok := i.CacheNodes[node.Name] + if !ok { + i.CacheNodes[node.Name] = node + continue + } + for key, value := range node.Props { + nowNode.Props[key] = value + } + } + for _, edge := range jsonObj.Edges { + src := edge.Dst + dst := edge.Src + if src != "" && dst != "" { + if _, ok := i.CacheEdges[src]; !ok { + i.CacheEdges[src] = make(map[string]Edge) + } + nowEdge, ok := i.CacheEdges[src][dst] + if !ok { + i.CacheEdges[src][dst] = edge + continue + } + for key, value := range edge.Props { + nowEdge.Props[key] = value + } + } + } +} + +// todo: get space schema +func (i *ImportJob) GetPrompt(text string) string { + i.Prompt = strings.ReplaceAll(i.LLMJob.PromptTemplate, "{spaceSchema}", i.LLMJob.SpaceSchemaString) + i.Prompt = strings.ReplaceAll(i.Prompt, "{text}", text) + return i.Prompt +} + +func (i *ImportJob) QueryBlocks(blocks []string) error { + job := i.LLMJob + maxBlocks := config.GetConfig().LLM.MaxBlockSize + for index, block := range blocks { + if index >= maxBlocks { + break + } + if IsRunningJobStopped(job.JobID) { + return nil + } + prompt := i.GetPrompt(block) + text, err := i.Query(prompt) + if err != nil { + i.WriteLogFile(fmt.Sprintf("query error: %v", err), "error") + continue + } + i.ParseText(text) + // update process + ratio := float64(index+1) / float64(len(blocks)) + i.Process.Ratio = 0.1 + ratio*0.6 + i.Process.CurrentSize = int((ratio * float64(i.Process.TotalSize))) + } + return nil +} + +func (i *ImportJob) Query(prompt string) (string, error) { + i.WriteLogFile(fmt.Sprintf("start query, prompt: %s", prompt), "info") + messages := make([]map[string]any, 0) + messages = append(messages, map[string]any{ + "role": "user", + "content": prompt, + }) + res, err := Fetch(i.AuthData, map[string]any{ + "stream": false, + "messages": messages, + "max_tokens": i.LLMConfig.ContextLengthLimit, + }, func(str string) {}) + if err != nil { + return "", err + } + i.WriteLogFile(fmt.Sprintf("query success, res: %v", res), "info") + text := res["choices"].([]any)[0].(map[string]any)["message"].(map[string]any)["content"].(string) + i.Process.PromptTokens += int(res["usage"].(map[string]any)["prompt_tokens"].(float64)) + i.Process.CompletionTokens += int(res["usage"].(map[string]any)["completion_tokens"].(float64)) + return text, nil +} + +func (i *ImportJob) MakeGQLFile() ([]string, error) { + i.WriteLogFile(fmt.Sprintf("start make gql file, nodes length: %d, edges length: %d", len(i.CacheNodes), len(i.CacheEdges)), "info") + gqls := make([]string, 0) + spaceVIDType := i.Schema.VidType + // if spaceVIDType == "FIXED_STRING(32)" { + regex := regexp.MustCompile(`\((\d+)\)`) + match := regex.FindStringSubmatch(spaceVIDType) + vidLength := 128 + if len(match) > 1 { + len, err := strconv.Atoi(match[1]) + if err == nil { + vidLength = len + } + } + isVidString := strings.Contains(spaceVIDType, "STRING") + for _, v := range i.CacheNodes { + typ := v.Type + name := v.Name + if isVidString { + if len(name) > vidLength { + name = name[:vidLength] + } + } + + typeSchema, ok := i.SchemaMap[typ] + if !ok { + continue + } + props := v.Props + propsStr := "" + valueStr := "" + + for key, field := range typeSchema { + value, ok := props[key] + if !ok { + if field.Nullable { + continue + } else { + value = "" + } + } + + if propsStr != "" { + propsStr += "," + } + propsStr += fmt.Sprintf("`%s`", key) + if valueStr != "" { + valueStr += "," + } + valueStr += fmt.Sprintf("`%v`", value) + } + + gql := fmt.Sprintf("INSERT VERTEX `%s` ({props}) VALUES \"%s\":({value});", typ, name) + gql = strings.ReplaceAll(gql, "{props}", propsStr) + gql = strings.ReplaceAll(gql, "{value}", propsStr) + gqls = append(gqls, gql) + } + + for _, src := range i.CacheEdges { + for _, dst := range src { + propsName := "" + propsValue := "" + typeSchema, ok := i.SchemaMap[dst.EdgeType] + if !ok { + continue + } + for key, field := range typeSchema { + value, ok := dst.Props[key] + if !ok { + if field.Nullable { + continue + } else { + value = "" + } + } + if propsName != "" { + propsName += "," + } + propsName += fmt.Sprintf("`%s`", key) + if propsValue != "" { + propsValue += "," + } + propsValue += fmt.Sprintf("\"%v\"", value) + } + gql := fmt.Sprintf("INSERT EDGE `%s` (%s) VALUES \"%s\"->\"%s\":(%s);", dst.EdgeType, propsName, dst.Src, dst.Dst, propsValue) + gqls = append(gqls, gql) + } + } + + gqlStr := strings.Join(gqls, "\n") + filePath := filepath.Join(config.GetConfig().LLM.GQLPath, fmt.Sprintf("%s/%s.gql", i.LLMJob.JobID, i.LLMJob.File)) + file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return nil, err + } + defer file.Close() + _, err = file.WriteString(gqlStr) + if err != nil { + return nil, err + } + i.WriteLogFile(fmt.Sprintf("make gql file success, gqls length: %d", len(gqls)), "info") + return gqls, nil +} + +func (i *ImportJob) RunGQLFile(gqls []string) error { + i.WriteLogFile(fmt.Sprintf("start run gql, gqls length: %d", len(gqls)), "info") + batchSize := config.GetConfig().LLM.GQLBatchSize + for index := 0; index < len(gqls); index += batchSize { + maxEnd := index + batchSize + if maxEnd > len(gqls) { + maxEnd = len(gqls) + } + res, err := client.Execute(i.NSID, i.LLMJob.Space, gqls[index:maxEnd]) + if err != nil { + i.WriteLogFile(fmt.Sprintf("run gql error: %v,index:%d,gqls:%v", err, index, gqls[index:maxEnd]), "error") + } else { + errors := make([]string, 0) + success := make([]string, 0) + for _, r := range res { + if r.Error != nil { + // i.WriteLogFile(fmt.Sprintf("run gql error: %v,gql:%s", r.Error, r.Gql), "error") + errors = append(errors, fmt.Sprintf("%s @error: %v", r.Gql, r.Error)) + } else { + success = append(success, r.Gql) + } + } + if len(success) > 0 { + i.WriteLogFile(fmt.Sprintf("run gql success:\n %v ", success), "info") + } + if len(errors) > 0 { + i.WriteLogFile(fmt.Sprintf("run gql error:\n %v ", errors), "error") + } + } + } + return nil +} diff --git a/server/api/studio/pkg/llm/job.go b/server/api/studio/pkg/llm/job.go new file mode 100644 index 00000000..a6d3779e --- /dev/null +++ b/server/api/studio/pkg/llm/job.go @@ -0,0 +1,77 @@ +package llm + +import ( + "sync" + + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" + "github.com/zeromicro/go-zero/core/logx" + "gorm.io/gorm" +) + +var ( + RunningJobMap = make(map[string]*db.LLMJob) + mu sync.Mutex +) + +func RunJobs(jobs []*db.LLMJob, JobRunnerMap map[string]func(job *db.LLMJob)) { + for _, job := range jobs { + job.Status = db.LLMStatusRunning + err := db.CtxDB.Save(job).Error + if err != nil { + logx.Errorf("failed to update job status: %v", err) + } + mu.Lock() + RunningJobMap[job.JobID] = job + mu.Unlock() + go func(job *db.LLMJob) { + defer func() { + mu.Lock() + delete(RunningJobMap, job.JobID) + mu.Unlock() + }() + runner := JobRunnerMap[job.JobType] + if runner != nil { + runner(job) + } + }(job) + } +} + +func IsRunningJobStopped(jobID string) bool { + mu.Lock() + defer mu.Unlock() + job, ok := RunningJobMap[jobID] + if ok { + return job.Status == db.LLMStatusCancel + } + return false +} + +func ChangeRunningJobStatus(jobID string, status db.LLMStatus) { + mu.Lock() + defer mu.Unlock() + job, ok := RunningJobMap[jobID] + if ok { + job.Status = status + } +} + +func GetRunningJob(jobID string) *db.LLMJob { + mu.Lock() + defer mu.Unlock() + job, ok := RunningJobMap[jobID] + if ok { + return job + } + return nil +} + +func GetPendingJobs() []*db.LLMJob { + var jobs []*db.LLMJob + err := db.CtxDB.Where("status = ?", db.LLMStatusPending).Find(&jobs).Error + if err != nil && err != gorm.ErrRecordNotFound { + logx.Errorf("failed to get pending jobs: %v", err) + } + + return jobs +} diff --git a/server/api/studio/pkg/llm/llm_test.go b/server/api/studio/pkg/llm/llm_test.go new file mode 100644 index 00000000..5ada8086 --- /dev/null +++ b/server/api/studio/pkg/llm/llm_test.go @@ -0,0 +1,41 @@ +package llm + +import ( + "regexp" + "strconv" + "testing" + + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" +) + +func TestImportJob_ReadPDFFile(t *testing.T) { + // Create a mock ImportJob instance + job := &ImportJob{ + LLMJob: &db.LLMJob{ + File: "data/llm/test.pdf", + }, + Process: &db.Process{}, + } + + // Call the ReadPDFFile method + result, err := job.ReadFile(job.LLMJob.File) + // Check if there was an error + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + //print pdf content + t.Log(result) +} + +func TestTypeLength(t *testing.T) { + regex := regexp.MustCompile(`\((\d+)\)`) + match := regex.FindStringSubmatch("FIXED_STRING(32)") + if len(match) > 1 { + t.Log(match[1]) + } + len, err := strconv.Atoi(match[1]) + if err != nil { + t.Log(err) + } + t.Log(len) +} diff --git a/server/api/studio/pkg/llm/schedule.go b/server/api/studio/pkg/llm/schedule.go new file mode 100644 index 00000000..b9a4c497 --- /dev/null +++ b/server/api/studio/pkg/llm/schedule.go @@ -0,0 +1,18 @@ +package llm + +import ( + "time" + + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" +) + +func InitSchedule() { + // TODO + for { + jobs := GetPendingJobs() + RunJobs(jobs, map[string]func(job *db.LLMJob){ + "file": RunFileJob, + }) + time.Sleep(5 * time.Second) + } +} diff --git a/server/api/studio/pkg/llm/transformer/openai.go b/server/api/studio/pkg/llm/transformer/openai.go new file mode 100644 index 00000000..a03be1ed --- /dev/null +++ b/server/api/studio/pkg/llm/transformer/openai.go @@ -0,0 +1,75 @@ +package transformer + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" +) + +type Handler interface { + HandleRequest(req map[string]any, config *db.LLMConfig) (*http.Request, error) + HandleResponse(resp *http.Response, callback func(str string)) (map[string]any, error) +} + +type OpenAI struct { +} + +func (o *OpenAI) HandleRequest(req map[string]any, config *db.LLMConfig) (*http.Request, error) { + // Convert the request parameters to a JSON string + reqJSON, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("failed to convert request parameters to JSON: %v", err) + } + + // Create an HTTP request + httpReq, err := http.NewRequest("POST", config.URL, strings.NewReader(string(reqJSON))) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP request: %v", err) + } + + // Set the Content-Type and Authorization headers + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+config.Key) + httpReq.Header.Set("api-key", config.Key) + return httpReq, nil +} + +func (o *OpenAI) HandleResponse(resp *http.Response, callback func(str string)) (map[string]any, error) { + // Check if the response is a server-sent event str + if resp.Header.Get("Content-Type") == "text/event-stream" { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "data:") { + event := strings.TrimPrefix(line, "data:") + event = strings.TrimSpace(event) + callback(event) + if strings.Contains(line, "[DONE]") { + break + } + } + } + if err := scanner.Err(); err != nil { + fmt.Println("reading standard input:", err) + } + return nil, nil + } + // Read the response data + var respData map[string]any + + // Read body text + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %s %v", string(bodyBytes), err) + } + err = json.Unmarshal(bodyBytes, &respData) + if err != nil { + return nil, fmt.Errorf("failed to parse response data: %s %v", string(bodyBytes), err) + } + return respData, nil +} diff --git a/server/api/studio/pkg/llm/transformer/qwen.go b/server/api/studio/pkg/llm/transformer/qwen.go new file mode 100644 index 00000000..2a87d7c9 --- /dev/null +++ b/server/api/studio/pkg/llm/transformer/qwen.go @@ -0,0 +1,99 @@ +package transformer + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strings" + + db "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/model" +) + +type Qwen struct { +} + +func (o *Qwen) HandleRequest(req map[string]any, config *db.LLMConfig) (*http.Request, error) { + configs := make(map[string]any) + qwenReq := make(map[string]any) + err := json.Unmarshal([]byte(config.Config), &configs) + if err == nil { + qwenReq["model"] = configs["model"] + } + input := make(map[string]any) + input["prompt"] = req["prompt"] + input["messages"] = req["messages"] + params := make(map[string]any) + for k, v := range req { + if k != "prompt" && k != "messages" { + params[k] = v + } + } + params["result_format"] = "message" + qwenReq["input"] = input + qwenReq["parameters"] = params + + // Convert the request parameters to a JSON string + reqJSON, err := json.Marshal(qwenReq) + if err != nil { + return nil, fmt.Errorf("failed to convert request parameters to JSON: %v", err) + } + + // Create an HTTP request + httpReq, err := http.NewRequest("POST", config.URL, strings.NewReader(string(reqJSON))) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP request: %v", err) + } + + // Set the Content-Type and Authorization headers + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+config.Key) + httpReq.Header.Set("api-key", config.Key) + if req["stream"] == true { + httpReq.Header.Set("Accept", "text/event-stream") + } + return httpReq, nil +} + +func (o *Qwen) HandleResponse(resp *http.Response, callback func(str string)) (map[string]any, error) { + // Check if the response is a server-sent event str + if strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "data:") { + event := strings.TrimPrefix(line, "data:") + event = strings.TrimSpace(event) + callback(event) + if strings.Contains(line, "[DONE]") { + break + } + } + } + if err := scanner.Err(); err != nil { + fmt.Println("reading standard input:", err) + } + return nil, nil + } + // Read the response data + var respData map[string]any + + // Read body text + bodyBytes, err := io.ReadAll(resp.Body) + log.Print("bodyBytes: ", string(bodyBytes)) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %s %v", string(bodyBytes), err) + } + err = json.Unmarshal(bodyBytes, &respData) + if err != nil { + return nil, fmt.Errorf("failed to parse response data: %s %v", string(bodyBytes), err) + } + + respData["choices"] = respData["output"].(map[string]any)["choices"] + usage := respData["usage"].(map[string]any) + usage["completion_tokens"] = usage["output_tokens"] + usage["prompt_tokens"] = usage["input_tokens"] + return respData, nil +} diff --git a/server/api/studio/pkg/pdf/reader.go b/server/api/studio/pkg/pdf/reader.go new file mode 100644 index 00000000..c00c8cb5 --- /dev/null +++ b/server/api/studio/pkg/pdf/reader.go @@ -0,0 +1,83 @@ +package pdf + +import ( + "fmt" + "regexp" + "sort" + "strings" + "unicode/utf8" + + "github.com/ledongthuc/pdf" +) + +func ReadPDFFile(filePath string) (string, error) { + + f, r, err := pdf.Open(filePath) + if err != nil { + return "", fmt.Errorf("open pdf file error: %v", err) + } + defer f.Close() + texts := make([]pdf.Text, 0) + fontSizeArr := make(map[float64]bool, 0) + for j := 1; j <= r.NumPage(); j++ { + page := r.Page(j) + if page.V.IsNull() { + continue + } + content := page.Content() + // read all text with fontSize + var lastTextStyle pdf.Text + for _, text := range content.Text { + if isSameSentence(lastTextStyle, text) { + // check utf code + if utf8.ValidString(text.S) { + lastTextStyle.S += text.S + } + } else { + texts = append(texts, lastTextStyle) + fontSizeArr[lastTextStyle.FontSize] = true + lastTextStyle = text + } + } + if lastTextStyle.S != "" { + texts = append(texts, lastTextStyle) + } + } + // sort fontSize + fontSizes := make([]float64, 0) + for fontSize := range fontSizeArr { + fontSizes = append(fontSizes, fontSize) + } + sort.Sort(sort.Reverse(sort.Float64Slice(fontSizes))) + sortFontSizeMap := make(map[float64]int) + for index, fontSize := range fontSizes { + sortFontSizeMap[fontSize] = index + 1 + } + + //read all text with fontSize + str := "" + for _, text := range texts { + text.S = strings.Trim(text.S, " ") + regex := regexp.MustCompile(`(\n(\n|\s){1,})|\.{6,}`) + text.S = regex.ReplaceAllString(text.S, "\n") + text.S = strings.ReplaceAll(text.S, "�", " ") + if len(text.S) == 0 || text.S == " " { + continue + } + fontSize := text.FontSize + index, ok := sortFontSizeMap[fontSize] + if !ok { + index = len(fontSizes) - 1 + } + if index < 4 { + str += fmt.Sprintf("%s\n", index, text.S, index) + } else { + str += text.S + "\n" + } + } + return str, nil +} + +func isSameSentence(text1 pdf.Text, text2 pdf.Text) bool { + return text1.FontSize == text2.FontSize && text1.Font == text2.Font +} diff --git a/server/api/studio/pkg/utils/file.go b/server/api/studio/pkg/utils/file.go index 36803b6e..2b9a61ba 100644 --- a/server/api/studio/pkg/utils/file.go +++ b/server/api/studio/pkg/utils/file.go @@ -28,8 +28,8 @@ func ReadPartFile(path string) ([]string, error) { defer file.Close() const ( - TopLineNum = 50 - BottomLineNum = 50 + TopLineNum = 1000 + BottomLineNum = 1000 ) var ( topLines []string diff --git a/server/api/studio/pkg/ws/middlewares/llm/receiver.go b/server/api/studio/pkg/ws/middlewares/llm/receiver.go new file mode 100644 index 00000000..ecebca34 --- /dev/null +++ b/server/api/studio/pkg/ws/middlewares/llm/receiver.go @@ -0,0 +1,85 @@ +package llm + +import ( + "encoding/json" + "fmt" + "log" + "strings" + "time" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/auth" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/llm" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ws/utils" +) + +func Middleware(next utils.TNext) utils.TNext { + return func(msgReceived *utils.MessageReceive, c *utils.Client) *utils.MessagePost { + if next == nil || msgReceived == nil { + return nil + } + if msgReceived.Body.MsgType != "llm" { + return next(msgReceived, c) + } + req := msgReceived.Body.Content + clientInfo, ok := c.GetClientInfo().(*auth.AuthData) + if !ok { + sendError(fmt.Errorf("invalid client info"), msgReceived, c) + return nil + } + data, err := llm.Fetch(clientInfo, req, func(str string) { + content := map[string]any{} + if strings.Contains(str, "[DONE]") { + content["done"] = true + } else if err := json.Unmarshal([]byte(str), &content); err != nil { + sendError(fmt.Errorf(err.Error(), str), msgReceived, c) + return + } + sendMsg(c, msgReceived, content) + }) + if err != nil { + sendError(err, msgReceived, c) + } + if data != nil { + sendMsg(c, msgReceived, data) + } + return nil + } +} + +func sendMsg(c *utils.Client, msgReceived *utils.MessageReceive, content map[string]any) { + msg := utils.MessagePost{ + Header: utils.MessagePostHeader{ + MsgId: msgReceived.Header.MsgId, + SendTime: time.Now().UnixMilli(), + }, + Body: utils.MessagePostBody{ + MsgType: msgReceived.Body.MsgType, + Content: map[string]any{ + "code": 0, + "message": content, + }, + }, + } + jsonMsg, err := json.Marshal(msg) + if err != nil { + log.Printf("send llm message error: %v", err) + return + } + c.SendMessage(jsonMsg) +} + +func sendError(err error, msgReceived *utils.MessageReceive, c *utils.Client) { + c.Conn.WriteJSON(utils.MessagePost{ + Header: utils.MessagePostHeader{ + MsgId: msgReceived.Header.MsgId, + SendTime: time.Now().UnixMilli(), + }, + Body: utils.MessagePostBody{ + MsgType: msgReceived.Body.MsgType, + Content: map[string]any{ + "code": 1, + "message": err.Error(), + }, + }, + }) +} diff --git a/server/api/studio/pkg/ws/utils/hub.go b/server/api/studio/pkg/ws/utils/hub.go index f0eabb78..86caf3ed 100644 --- a/server/api/studio/pkg/ws/utils/hub.go +++ b/server/api/studio/pkg/ws/utils/hub.go @@ -47,15 +47,6 @@ func (h *Hub) Run() { } } h.mu.Unlock() - // case message := <-h.broadcast: - // for client := range h.clients { - // select { - // case client.send <- message: - // default: - // close(client.send) - // delete(h.clients, client) - // } - // } } } } diff --git a/server/api/studio/pkg/ws/ws.go b/server/api/studio/pkg/ws/ws.go index 8dff432e..3ef1104a 100644 --- a/server/api/studio/pkg/ws/ws.go +++ b/server/api/studio/pkg/ws/ws.go @@ -6,6 +6,7 @@ import ( "github.com/gorilla/websocket" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/auth" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ws/middlewares/batch_ngql" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ws/middlewares/llm" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ws/middlewares/logger" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ws/middlewares/ngql" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ws/utils" @@ -41,6 +42,7 @@ func ServeWebSocket(hub *utils.Hub, w http.ResponseWriter, r *http.Request, clie logger.Middleware, batch_ngql.Middleware, ngql.Middleware, + llm.Middleware, }) client.Serve() } diff --git a/server/api/studio/restapi/import.api b/server/api/studio/restapi/import.api index 81f3f5cf..e8a2daa0 100644 --- a/server/api/studio/restapi/import.api +++ b/server/api/studio/restapi/import.api @@ -171,6 +171,7 @@ type ( UpdateTime int64 `json:"updateTime"` Stats ImportTaskStats `json:"stats"` RawConfig string `json:"rawConfig"` + LLMJob interface{} `json:"llmJob"` } ImportTaskStats { diff --git a/server/api/studio/restapi/llm.api b/server/api/studio/restapi/llm.api new file mode 100644 index 00000000..0b5eb494 --- /dev/null +++ b/server/api/studio/restapi/llm.api @@ -0,0 +1,84 @@ +syntax = "v1" + +type ( + LLMRequest { + Data map[string]interface{} `json:"data"` + } + LLMResponse { + Data interface{} `json:"data"` + } + LLMConfigRequest { + URL string `json:"url"` + Key string `json:"key,optional"` + APIType string `json:"apiType"` + MaxContextLength int `json:"maxContextLength"` + Config string `json:"config,optional"` + } + + LLMImportRequest { + Space string `json:"space"` + File string `json:"file,optional"` + FilePath string `json:"filePath,optional"` + Type string `json:"type"` + PromptTemplate string `json:"promptTemplate"` + SpaceSchemaString string `json:"spaceSchemaString"` + } + + LLMImportJobsRequest { + Page int `json:"page"` + PageSize int `json:"pageSize"` + Space string `json:"space,optional"` + } + + LLMImportLogRequest { + JobID string `json:"jobId"` + } + + HandleLLMImportRequest { + JobID string `json:"jobId"` + Action string `path:"action"` + } + + DeleteLLMImportRequest { + JobID string `path:"jobId"` + } + DownloadLLMImportNgqlRequest { + JobID string `json:"jobId"` + } +) + +@server( + group: llm +) + +service studio-api { + + @doc "LLMProxy" + @handler LLMProxy + post /api/llm (LLMRequest) returns (LLMResponse) + + @doc "LLMConfig" + @handler LLMConfig + post /api/config/llm(LLMConfigRequest) + + @handler GetLLMConfig + get /api/config/llm returns (LLMResponse) + + @handler AddLLMImportJob + post /api/llm/import/job (LLMImportRequest) returns (LLMResponse) + + @handler GetLLMImportJobs + get /api/llm/import/jobs (LLMImportJobsRequest) returns (LLMResponse) + + @handler GetLLMImportJobLog + get /api/llm/import/job/log (LLMImportLogRequest) returns (LLMResponse) + + @handler HandleLLMImportJob + post /api/llm/import/job/:action (HandleLLMImportRequest) returns (LLMResponse) + + @handler DeleteLLMImportJob + delete /api/llm/import/job/:jobId (DeleteLLMImportRequest) returns (LLMResponse) + + @handler DownloadLLMImportNgql + get /api/llm/import/ngql (DownloadLLMImportNgqlRequest) returns (LLMResponse) +} \ No newline at end of file diff --git a/server/api/studio/restapi/studio.api b/server/api/studio/restapi/studio.api index ebbd0950..f3710145 100644 --- a/server/api/studio/restapi/studio.api +++ b/server/api/studio/restapi/studio.api @@ -14,4 +14,5 @@ import ( "schema.api" "favorite.api" "datasource.api" + "llm.api" ) \ No newline at end of file diff --git a/server/api/studio/studio.go b/server/api/studio/studio.go index 5423de7e..03938770 100644 --- a/server/api/studio/studio.go +++ b/server/api/studio/studio.go @@ -12,6 +12,7 @@ import ( "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/auth" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/client" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/llm" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/logging" studioMiddleware "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/middleware" "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/server" @@ -103,7 +104,7 @@ func main() { httpx.SetErrorHandler(func(err error) (int, interface{}) { return svcCtx.ResponseHandler.GetStatusBody(nil, nil, err) }) - + go llm.InitSchedule() fmt.Printf("Starting server at %s:%d...\n", c.Host, c.Port) server.Start() } diff --git a/server/go.mod b/server/go.mod index 8f65ebd5..3c8e9d25 100644 --- a/server/go.mod +++ b/server/go.mod @@ -15,11 +15,13 @@ require ( github.com/agiledragon/gomonkey/v2 v2.9.0 github.com/aws/aws-sdk-go v1.44.217 github.com/golang/mock v1.6.0 + github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 github.com/pkg/sftp v1.13.5 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.8.0 github.com/vesoft-inc/nebula-go/v3 v3.5.0 gopkg.in/yaml.v3 v3.0.1 + gorm.io/datatypes v1.2.0 gorm.io/driver/mysql v1.5.1 ) @@ -40,6 +42,8 @@ require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect + github.com/hhrutter/lzw v1.0.0 // indirect + github.com/hhrutter/tiff v1.0.1 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.0.0 // indirect @@ -49,18 +53,27 @@ require ( github.com/jlaffaye/ftp v0.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/panjf2000/ants v1.2.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect + github.com/unidoc/pkcs7 v0.2.0 // indirect + github.com/unidoc/timestamp v0.0.0-20200412005513-91597fd3793a // indirect + github.com/unidoc/unitype v0.2.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect + golang.org/x/image v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) require ( github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/digitorus/pdf v0.1.2 github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.0 // indirect @@ -70,10 +83,12 @@ require ( github.com/golang-jwt/jwt/v4 v4.2.0 github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/jung-kurt/gofpdf v1.16.2 github.com/justinas/alice v1.2.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/openzipkin/zipkin-go v0.4.0 // indirect + github.com/pdfcpu/pdfcpu v0.5.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -81,6 +96,7 @@ require ( github.com/prometheus/procfs v0.7.3 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/unidoc/unipdf/v3 v3.52.0 go.opentelemetry.io/otel v1.3.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.3.0 // indirect go.opentelemetry.io/otel/exporters/zipkin v1.3.0 // indirect @@ -88,10 +104,10 @@ require ( go.opentelemetry.io/otel/trace v1.3.0 // indirect go.uber.org/automaxprocs v1.4.0 // indirect go.uber.org/zap v1.23.0 - golang.org/x/crypto v0.5.0 - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/crypto v0.14.0 + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/grpc v1.46.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/server/go.sum b/server/go.sum index f3597b68..ab914912 100644 --- a/server/go.sum +++ b/server/go.sum @@ -73,6 +73,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -102,6 +103,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/digitorus/pdf v0.1.2 h1:RjYEJNbiV6Kcn8QzRi6pwHuOaSieUUrg4EZo4b7KuIQ= +github.com/digitorus/pdf v0.1.2/go.mod h1:05fDDJhPswBRM7GTfqCxNiDyeNcN0f+IobfOAl5pdXw= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -181,6 +184,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -268,9 +273,16 @@ github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2I github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hhrutter/lzw v1.0.0 h1:laL89Llp86W3rRs83LvKbwYRx6INE8gDn0XNb1oXtm0= +github.com/hhrutter/lzw v1.0.0/go.mod h1:2HC6DJSn/n6iAZfgM3Pg+cP1KxeWc3ezG8bBqW5+WEo= +github.com/hhrutter/tiff v1.0.1 h1:MIus8caHU5U6823gx7C6jrfoEvfSTGtEFRiM8/LOzC0= +github.com/hhrutter/tiff v1.0.1/go.mod h1:zU/dNgDm0cMIa8y8YwcYBeuEEveI4B0owqHyiPpJPHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -304,12 +316,16 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5PtRc= +github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0= github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo= github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNEe7i7qA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -324,6 +340,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -334,12 +352,15 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -374,7 +395,10 @@ github.com/openzipkin/zipkin-go v0.4.0 h1:CtfRrOVZtbDj8rt1WXjklw0kqqJQwICrCKmlfU github.com/openzipkin/zipkin-go v0.4.0/go.mod h1:4c3sLeE8xjNqehmF5RpAFLPLJxXscc0R4l6Zg0P1tTQ= github.com/panjf2000/ants v1.2.1 h1:IlhLREssFi+YFOITnHdH3FHhulY6WDS0OB9e7+3fMHk= github.com/panjf2000/ants v1.2.1/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= +github.com/pdfcpu/pdfcpu v0.5.0 h1:F3wC4bwPbaJM+RPgm1D0Q4SAUwxElw7BhwNvL3iPgDo= +github.com/pdfcpu/pdfcpu v0.5.0/go.mod h1:UPcHdWcMw1V6Bo5tcWHd3jZfkG8cwUwrJkQOlB6o+7g= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/phpdave11/gofpdi v1.0.7/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -411,6 +435,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -419,6 +445,7 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= @@ -426,7 +453,9 @@ github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrY github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -451,6 +480,15 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/unidoc/pkcs7 v0.0.0-20200411230602-d883fd70d1df/go.mod h1:UEzOZUEpJfDpywVJMUT8QiugqEZC29pDq7kdIZhWCr8= +github.com/unidoc/pkcs7 v0.2.0 h1:0Y0RJR5Zu7OuD+/l7bODXARn6b8Ev2G4A8lI4rzy9kg= +github.com/unidoc/pkcs7 v0.2.0/go.mod h1:UEzOZUEpJfDpywVJMUT8QiugqEZC29pDq7kdIZhWCr8= +github.com/unidoc/timestamp v0.0.0-20200412005513-91597fd3793a h1:RLtvUhe4DsUDl66m7MJ8OqBjq8jpWBXPK6/RKtqeTkc= +github.com/unidoc/timestamp v0.0.0-20200412005513-91597fd3793a/go.mod h1:j+qMWZVpZFTvDey3zxUkSgPJZEX33tDgU/QIA0IzCUw= +github.com/unidoc/unipdf/v3 v3.52.0 h1:yhgUhqDfT2oBiidsixpWFIVF1tZf5dvKq7rkMoXmRQk= +github.com/unidoc/unipdf/v3 v3.52.0/go.mod h1:nsyu8C7iOhmASFPmYkwQ6nFhSnIpHOKxQeDfaS80m38= +github.com/unidoc/unitype v0.2.1 h1:x0jMn7pB/tNrjEVjy3Ukpxo++HOBQaTCXcTYFA6BH3w= +github.com/unidoc/unitype v0.2.1/go.mod h1:mafyug7zYmDOusqa7G0dJV45qp4b6TDAN+pHN7ZUIBU= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -521,8 +559,10 @@ golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -535,6 +575,9 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -601,6 +644,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -652,6 +696,7 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -680,6 +725,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -696,6 +743,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -756,6 +805,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -874,10 +924,14 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/datatypes v1.2.0 h1:5YT+eokWdIxhJgWHdrb2zYUimyk0+TaFth+7a0ybzco= +gorm.io/datatypes v1.2.0/go.mod h1:o1dh0ZvjIjhH/bngTpypG6lVRJ5chTBxE09FH/71k04= gorm.io/driver/mysql v1.5.1 h1:WUEH5VF9obL/lTtzjmML/5e6VfFR/788coz2uaVCAZw= gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5o= +gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U= gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc= gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= +gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= diff --git a/tsconfig.json b/tsconfig.json index c06f8d07..d9afeb31 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -32,8 +32,19 @@ ], "baseUrl": ".", "paths": { - "@public/*": ["public/*"], - "@app/*": ["app/*"] + "@public/*": [ + "public/*" + ], + "@app/*": [ + "app/*" + ] } - } -} + }, + "include": [ + "./**/*.ts", + "./**/*.tsx", + "./**/*.js", + "./**/*.json", + "scripts/readPdf.mjs" + ] +} \ No newline at end of file diff --git a/vite.config.ts b/vite.config.ts index 2dc8104e..c220cf2b 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -51,6 +51,7 @@ export default defineConfig({ server: { port: 7001, open: true, + host: '0.0.0.0', proxy: { '/api-nebula': { target: `http://${proxyHost}`,