From 888d3efc16e652ef3aec6d321248fa59ae5a8e41 Mon Sep 17 00:00:00 2001 From: spacedragon Date: Fri, 19 Jul 2019 14:58:55 +0800 Subject: [PATCH] [Code] implement filtree api by using isogit --- x-pack/legacy/plugins/code/model/commit.ts | 1 + .../plugins/code/server/git_operations.ts | 316 +++++++++--------- .../code/server/lsp/lsp_test_runner.ts | 22 +- .../legacy/plugins/code/server/routes/file.ts | 3 +- 4 files changed, 167 insertions(+), 175 deletions(-) diff --git a/x-pack/legacy/plugins/code/model/commit.ts b/x-pack/legacy/plugins/code/model/commit.ts index 595d50e85da2a..f4d613d509624 100644 --- a/x-pack/legacy/plugins/code/model/commit.ts +++ b/x-pack/legacy/plugins/code/model/commit.ts @@ -11,6 +11,7 @@ export interface CommitInfo { author: string; id: string; parents: string[]; + treeId: string; } export interface ReferenceInfo { diff --git a/x-pack/legacy/plugins/code/server/git_operations.ts b/x-pack/legacy/plugins/code/server/git_operations.ts index 49d6aabd422a3..d2adffbccaab9 100644 --- a/x-pack/legacy/plugins/code/server/git_operations.ts +++ b/x-pack/legacy/plugins/code/server/git_operations.ts @@ -13,8 +13,6 @@ import { Error as NodeGitError, Oid, Repository, - Tree, - TreeEntry, } from '@elastic/nodegit'; import Boom from 'boom'; import LruCache from 'lru-cache'; @@ -23,10 +21,9 @@ import * as fs from 'fs'; import * as isogit from 'isomorphic-git'; import { CommitDescription, TreeDescription } from 'isomorphic-git'; import { isBinaryFileSync } from 'isbinaryfile'; - import { GitBlame } from '../common/git_blame'; import { CommitDiff, Diff, DiffKind } from '../common/git_diff'; -import { FileTree, FileTreeItemType, RepositoryUri, sortFileTree } from '../model'; +import { FileTree, FileTreeItemType, RepositoryUri } from '../model'; import { CommitInfo, ReferenceInfo, ReferenceType } from '../model/commit'; import { detectLanguage } from './utils/detect_language'; @@ -59,34 +56,22 @@ async function checkExists(func: () => Promise, message: string): Promise< return result; } -function entry2Tree(entry: TreeEntry): FileTree { - let type: FileTreeItemType; - switch (entry.filemode()) { - case TreeEntry.FILEMODE.LINK: - type = FileTreeItemType.Link; - break; - case TreeEntry.FILEMODE.COMMIT: - type = FileTreeItemType.Submodule; - break; - case TreeEntry.FILEMODE.TREE: - type = FileTreeItemType.Directory; - break; - case TreeEntry.FILEMODE.BLOB: - case TreeEntry.FILEMODE.EXECUTABLE: - type = FileTreeItemType.File; - break; - default: - // @ts-ignore - throw new Error('unreadable file'); - } +function entry2Tree(entry: isogit.TreeEntry, prefixPath: string = ''): FileTree { + const type: FileTreeItemType = GitOperations.mode2type(entry.mode); + const { path, oid } = entry; return { - name: entry.name(), - path: entry.path(), - sha1: entry.sha(), + name: path, + path: prefixPath ? prefixPath + '/' + path : path, + sha1: oid, type, }; } +interface Tree { + entries: isogit.TreeEntry[]; + gitdir: string; + oid: string; +} export class GitOperations { private REPO_LRU_CACHE_SIZE = 16; private REPO_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour; @@ -221,6 +206,18 @@ export class GitOperations { throw new Error('invalid path'); } } + + private static async isTextFile(gitdir: string, entry: isogit.TreeEntry) { + if (entry.type === 'blob') { + const type = GitOperations.mode2type(entry.mode); + if (type === FileTreeItemType.File) { + const blob = await isogit.readObject({ gitdir, oid: entry.oid, format: 'content' }); + return !isBinaryFileSync(blob.object as Buffer); + } + } + return false; + } + public async countRepoFiles(uri: RepositoryUri, revision: string): Promise { let count = 0; const commit = await this.getCommitOr404(uri, revision); @@ -229,19 +226,12 @@ export class GitOperations { const treeId = (commitObject.object as CommitDescription).tree; async function walk(oid: string) { - const { object } = await isogit.readObject({ gitdir, oid }); - const tree = object as TreeDescription; + const tree = await GitOperations.readTree(gitdir, oid); for (const entry of tree.entries) { if (entry.type === 'tree') { await walk(entry.oid); - } else if (entry.type === 'blob') { - const type = GitOperations.mode2type(entry.mode); - if (type === FileTreeItemType.File) { - const blob = await isogit.readObject({ gitdir, oid: entry.oid, format: 'content' }); - if (!isBinaryFileSync(blob.object as Buffer)) { - count++; - } - } + } else if (await GitOperations.isTextFile(gitdir, entry)) { + count++; } } } @@ -254,38 +244,48 @@ export class GitOperations { uri: RepositoryUri, revision: string ): Promise> { + const commit = await this.getCommitOr404(uri, revision); + const gitdir = this.repoDir(uri); + const commitObject = await isogit.readObject({ gitdir, oid: commit.id }); + const treeId = (commitObject.object as CommitDescription).tree; async function* walk(oid: string, prefix: string = ''): AsyncIterableIterator { - const { object } = await isogit.readObject({ gitdir, oid }); - const tree = object as TreeDescription; + const tree = await GitOperations.readTree(gitdir, oid); for (const entry of tree.entries) { const path = prefix ? `${prefix}/${entry.path}` : entry.path; if (entry.type === 'tree') { yield* walk(entry.oid, path); - } else if (entry.type === 'blob') { + } else if (await GitOperations.isTextFile(gitdir, entry)) { const type = GitOperations.mode2type(entry.mode); - if (type === FileTreeItemType.File) { - const blob = await isogit.readObject({ gitdir, oid: entry.oid, format: 'content' }); - if (!isBinaryFileSync(blob.object as Buffer)) { - yield { - name: entry.path, - type, - path, - repoUri: uri, - sha1: entry.oid, - } as FileTree; - } - } + yield { + name: entry.path, + type, + path, + repoUri: uri, + sha1: entry.oid, + } as FileTree; } } } - const commit = await this.getCommitOr404(uri, revision); - const gitdir = this.repoDir(uri); - const commitObject = await isogit.readObject({ gitdir, oid: commit.id }); - const treeId = (commitObject.object as CommitDescription).tree; return await walk(treeId); } + private static async readTree(gitdir: string, oid: string): Promise { + const { type, object } = await isogit.readObject({ gitdir, oid }); + if (type === 'commit') { + return await this.readTree(gitdir, (object as CommitDescription).tree); + } else if (type === 'tree') { + const tree = object as TreeDescription; + return { + entries: tree.entries, + gitdir, + oid, + } as Tree; + } else { + throw new Error(`${oid} is not a tree`); + } + } + static mode2type(mode: string): FileTreeItemType { switch (mode) { case '100755': @@ -311,7 +311,6 @@ export class GitOperations { * @param skip pagination parameter, skip how many nodes in each children. * @param limit pagination parameter, limit the number of node's children. * @param resolveParents whether the return value should always start from root - * @param childrenDepth how depth should the children walk. * @param flatten */ public async fileTree( @@ -321,54 +320,111 @@ export class GitOperations { skip: number = 0, limit: number = DEFAULT_TREE_CHILDREN_LIMIT, resolveParents: boolean = false, - childrenDepth: number = 1, flatten: boolean = false ): Promise { - const commit = await this.getCommit(uri, revision); - const tree = await commit.getTree(); - if (path === '/') { - path = ''; + const commit = await this.getCommitOr404(uri, revision); + const gitdir = this.repoDir(uri); + if (path.startsWith('/')) { + path = path.slice(1); } - const getRoot = async () => { - return await this.walkTree( - { - name: '', - path: '', - type: FileTreeItemType.Directory, - }, - tree, - [], - skip, - limit, - childrenDepth, - flatten - ); - }; - if (path) { - if (resolveParents) { - return this.walkTree( - await getRoot(), - tree, - path.split('/'), - skip, - limit, - childrenDepth, - flatten - ); - } else { - const entry = await checkExists( - () => Promise.resolve(tree.getEntry(path)), - `path ${path} does not exists.` + if (path.endsWith('/')) { + path = path.slice(0, -1); + } + + function type2item(type: string) { + switch (type) { + case 'blob': + return FileTreeItemType.File; + case 'tree': + return FileTreeItemType.Directory; + case 'commit': + return FileTreeItemType.Submodule; + default: + throw new Error(`unsupported file tree item type ${type}`); + } + } + + if (resolveParents) { + const root: FileTree = { + name: '', + path: '', + type: FileTreeItemType.Directory, + }; + const tree = await GitOperations.readTree(gitdir, commit.treeId); + await this.fillChildren(root, tree, { skip, limit, flatten }); + if (path) { + await this.resolvePath(root, tree, path.split('/'), { skip, limit, flatten }); + } + return root; + } else { + const obj = await isogit.readObject({ gitdir, oid: commit.id, filepath: path }); + const result: FileTree = { + name: path.split('/').pop() || '', + path, + type: type2item(obj.type!), + sha1: obj.oid, + }; + if (result.type === FileTreeItemType.Directory) { + await this.fillChildren( + result, + { + gitdir, + entries: (obj.object as TreeDescription).entries, + oid: obj.oid, + }, + { skip, limit, flatten } ); - if (entry.isDirectory()) { - const tree1 = await entry.getTree(); - return this.walkTree(entry2Tree(entry), tree1, [], skip, limit, childrenDepth, flatten); + } + return result; + } + } + + private async fillChildren( + result: FileTree, + { entries, gitdir }: Tree, + { skip, limit, flatten }: { skip: number; limit: number; flatten: boolean } + ) { + result.childrenCount = entries.length; + result.children = []; + for (const e of entries.slice(skip, Math.min(entries.length, skip + limit))) { + const child = entry2Tree(e, result.path); + result.children.push(child); + if (flatten && child.type === FileTreeItemType.Directory) { + const tree = await GitOperations.readTree(gitdir, e.oid); + if (tree.entries.length === 1) { + await this.fillChildren(child, tree, { skip, limit, flatten }); + } + } + } + } + + private async resolvePath( + result: FileTree, + tree: Tree, + paths: string[], + opt: { skip: number; limit: number; flatten: boolean } + ) { + const [path, ...rest] = paths; + for (const entry of tree.entries) { + if (entry.path === path) { + if (!result.children) { + result.children = []; + } + const child = entry2Tree(entry, result.path); + const idx = result.children.findIndex(i => i.sha1 === entry.oid); + if (idx < 0) { + result.children.push(child); } else { - return entry2Tree(entry); + result.children[idx] = child; + } + if (entry.type === 'tree') { + const subTree = await GitOperations.readTree(tree.gitdir, entry.oid); + await this.fillChildren(child, subTree, opt); + if (rest.length > 0) { + await this.resolvePath(child, subTree, rest, opt); + } } } - } else { - return getRoot(); } } @@ -521,68 +577,6 @@ export class GitOperations { return (await entry.getBlob()).content().toString('utf8'); } - private async walkTree( - fileTree: FileTree, - tree: Tree, - paths: string[], - skip: number, - limit: number, - childrenDepth: number = 1, - flatten: boolean = false - ): Promise { - const [path, ...rest] = paths; - fileTree.childrenCount = tree.entryCount(); - if (!fileTree.children) { - fileTree.children = []; - for (const e of tree.entries().slice(skip, limit)) { - const child = entry2Tree(e); - fileTree.children.push(child); - if (e.isDirectory()) { - const childChildrenCount = (await e.getTree()).entryCount(); - if ((childChildrenCount === 1 && flatten) || childrenDepth > 1) { - await this.walkTree( - child, - await e.getTree(), - [], - skip, - limit, - childrenDepth - 1, - flatten - ); - } - } - } - fileTree.children.sort(sortFileTree); - } - if (path) { - const entry = await checkExists( - () => Promise.resolve(tree.getEntry(path)), - `path ${fileTree.path}/${path} does not exists.` - ); - let child = entry2Tree(entry); - if (entry.isDirectory()) { - child = await this.walkTree( - child, - await entry.getTree(), - rest, - skip, - limit, - childrenDepth, - flatten - ); - } - const idx = fileTree.children.findIndex(c => c.name === entry.name()); - if (idx >= 0) { - // replace the entry in children if found - fileTree.children[idx] = child; - } else { - fileTree.children.push(child); - } - } - - return fileTree; - } - private async findCommit(repo: Repository, oid: string): Promise { try { return repo.getCommit(Oid.fromString(oid)); @@ -675,6 +669,7 @@ export class GitOperations { message: commit.message, updated: new Date(commit.committer.timestamp * 1000), parents: commit.parent, + treeId: commit.tree, } as CommitInfo; } else if (obj.type === 'tag') { const tag = obj.object as isogit.TagDescription; @@ -695,5 +690,6 @@ export function commitInfo(commit: Commit): CommitInfo { committer: commit.committer().name(), id: commit.sha().substr(0, 7), parents: commit.parents().map(oid => oid.toString().substring(0, 7)), + treeId: commit.treeId().tostrS(), }; } diff --git a/x-pack/legacy/plugins/code/server/lsp/lsp_test_runner.ts b/x-pack/legacy/plugins/code/server/lsp/lsp_test_runner.ts index 5190d485bd881..943fc7bd32134 100644 --- a/x-pack/legacy/plugins/code/server/lsp/lsp_test_runner.ts +++ b/x-pack/legacy/plugins/code/server/lsp/lsp_test_runner.ts @@ -20,7 +20,6 @@ import { TypescriptServerLauncher } from './ts_launcher'; import { GitOperations } from '../git_operations'; import { createTestServerOption } from '../test_utils'; import { ConsoleLoggerFactory } from '../utils/console_logger_factory'; -import { RepositoryUtils } from '../../common/repository_utils'; import { LspRequest } from '../../model'; import { Repo, RequestType } from '../../model/test_config'; @@ -191,18 +190,15 @@ export class LspTestRunner { private async getAllFile() { const gitOperator: GitOperations = new GitOperations(this.repo.path); try { - const fileTree = await gitOperator.fileTree( - '', - '', - 'HEAD', - 0, - Number.MAX_SAFE_INTEGER, - false, - Number.MAX_SAFE_INTEGER - ); - return RepositoryUtils.getAllFiles(fileTree).filter((filePath: string) => { - return filePath.endsWith(this.repo.language); - }); + const result: string[] = []; + const fileIterator = await gitOperator.iterateRepo('', 'HEAD'); + for await (const file of fileIterator) { + const filePath = file.path!; + if (filePath.endsWith(this.repo.language)) { + result.push(filePath); + } + } + return result; } catch (e) { console.error(`get files error: ${e}`); throw e; diff --git a/x-pack/legacy/plugins/code/server/routes/file.ts b/x-pack/legacy/plugins/code/server/routes/file.ts index 75c9fc7cc02ae..e1458a5c28844 100644 --- a/x-pack/legacy/plugins/code/server/routes/file.ts +++ b/x-pack/legacy/plugins/code/server/routes/file.ts @@ -41,7 +41,6 @@ export function fileRoute(server: CodeServerRouter, gitOps: GitOperations) { ? parseInt(queries.limit as string, 10) : DEFAULT_TREE_CHILDREN_LIMIT; const skip = queries.skip ? parseInt(queries.skip as string, 10) : 0; - const depth = queries.depth ? parseInt(queries.depth as string, 10) : 0; const withParents = 'parents' in queries; const flatten = 'flatten' in queries; const repoExist = await repoExists(req, uri); @@ -50,7 +49,7 @@ export function fileRoute(server: CodeServerRouter, gitOps: GitOperations) { } try { - return await gitOps.fileTree(uri, path, revision, skip, limit, withParents, depth, flatten); + return await gitOps.fileTree(uri, path, revision, skip, limit, withParents, flatten); } catch (e) { if (e.isBoom) { return e;