From ba5df8251775a71936e3c2a01b09b2049aea6a4e Mon Sep 17 00:00:00 2001 From: sdghchj Date: Sat, 26 Nov 2022 10:55:18 +0800 Subject: [PATCH] record token.FileSet for every file so that the position of parsing error can be acquired (#1393) --- packages.go | 32 ++++++++++-- packages_test.go | 29 +++++++---- parser.go | 19 ++----- parser_test.go | 127 ++++++++++++++++++----------------------------- types.go | 4 ++ 5 files changed, 104 insertions(+), 107 deletions(-) diff --git a/packages.go b/packages.go index c00ebd12c..9480acb17 100644 --- a/packages.go +++ b/packages.go @@ -1,6 +1,7 @@ package swag import ( + "fmt" "go/ast" goparser "go/parser" "go/token" @@ -19,6 +20,7 @@ type PackagesDefinitions struct { packages map[string]*PackageDefinitions uniqueDefinitions map[string]*TypeSpecDef parseDependency bool + debug Debugger } // NewPackagesDefinitions create object PackagesDefinitions. @@ -30,8 +32,19 @@ func NewPackagesDefinitions() *PackagesDefinitions { } } -// CollectAstFile collect ast.file. -func (pkgDefs *PackagesDefinitions) CollectAstFile(packageDir, path string, astFile *ast.File) error { +// ParseFile parse a source file. +func (pkgDefs *PackagesDefinitions) ParseFile(packageDir, path string, src interface{}) error { + // positions are relative to FileSet + fileSet := token.NewFileSet() + astFile, err := goparser.ParseFile(fileSet, path, src, goparser.ParseComments) + if err != nil { + return fmt.Errorf("failed to parse file %s, error:%+v", path, err) + } + return pkgDefs.collectAstFile(fileSet, packageDir, path, astFile) +} + +// collectAstFile collect ast.file. +func (pkgDefs *PackagesDefinitions) collectAstFile(fileSet *token.FileSet, packageDir, path string, astFile *ast.File) error { if pkgDefs.files == nil { pkgDefs.files = make(map[*ast.File]*AstFileInfo) } @@ -64,6 +77,7 @@ func (pkgDefs *PackagesDefinitions) CollectAstFile(packageDir, path string, astF } pkgDefs.files[astFile] = &AstFileInfo{ + FileSet: fileSet, File: astFile, Path: path, PackagePath: packageDir, @@ -73,9 +87,9 @@ func (pkgDefs *PackagesDefinitions) CollectAstFile(packageDir, path string, astF } // RangeFiles for range the collection of ast.File in alphabetic order. -func rangeFiles(files map[*ast.File]*AstFileInfo, handle func(filename string, file *ast.File) error) error { - sortedFiles := make([]*AstFileInfo, 0, len(files)) - for _, info := range files { +func (pkgDefs *PackagesDefinitions) RangeFiles(handle func(filename string, file *ast.File) error) error { + sortedFiles := make([]*AstFileInfo, 0, len(pkgDefs.files)) + for _, info := range pkgDefs.files { // ignore package path prefix with 'vendor' or $GOROOT, // because the router info of api will not be included these files. if strings.HasPrefix(info.PackagePath, "vendor") || strings.HasPrefix(info.Path, runtime.GOROOT()) { @@ -270,6 +284,14 @@ func (pkgDefs *PackagesDefinitions) evaluateAllConstVariables() { // EvaluateConstValue evaluate a const variable. func (pkgDefs *PackagesDefinitions) EvaluateConstValue(pkg *PackageDefinitions, cv *ConstVariable, recursiveStack map[string]struct{}) (interface{}, ast.Expr) { if expr, ok := cv.Value.(ast.Expr); ok { + defer func() { + if err := recover(); err != nil { + if fi, ok := pkgDefs.files[cv.File]; ok { + pos := fi.FileSet.Position(cv.Name.NamePos) + pkgDefs.debug.Printf("warning: failed to evaluate const %s at %s:%d:%d, %v", cv.Name.Name, fi.Path, pos.Line, pos.Column, err) + } + } + }() if recursiveStack == nil { recursiveStack = make(map[string]struct{}) } diff --git a/packages_test.go b/packages_test.go index fa736d6a8..60beca2f8 100644 --- a/packages_test.go +++ b/packages_test.go @@ -10,20 +10,30 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPackagesDefinitions_CollectAstFile(t *testing.T) { +func TestPackagesDefinitions_ParseFile(t *testing.T) { pd := PackagesDefinitions{} - assert.NoError(t, pd.CollectAstFile("", "", nil)) + packageDir := "github.com/swaggo/swag/testdata/simple" + assert.NoError(t, pd.ParseFile(packageDir, "testdata/simple/main.go", nil)) + assert.Equal(t, 1, len(pd.packages)) + assert.Equal(t, 1, len(pd.files)) +} + +func TestPackagesDefinitions_collectAstFile(t *testing.T) { + pd := PackagesDefinitions{} + fileSet := token.NewFileSet() + assert.NoError(t, pd.collectAstFile(fileSet, "", "", nil)) firstFile := &ast.File{ Name: &ast.Ident{Name: "main.go"}, } packageDir := "github.com/swaggo/swag/testdata/simple" - assert.NoError(t, pd.CollectAstFile(packageDir, "testdata/simple/"+firstFile.Name.String(), firstFile)) + assert.NoError(t, pd.collectAstFile(fileSet, packageDir, "testdata/simple/"+firstFile.Name.String(), firstFile)) assert.NotEmpty(t, pd.packages[packageDir]) absPath, _ := filepath.Abs("testdata/simple/" + firstFile.Name.String()) astFileInfo := &AstFileInfo{ + FileSet: fileSet, File: firstFile, Path: absPath, PackagePath: packageDir, @@ -31,14 +41,14 @@ func TestPackagesDefinitions_CollectAstFile(t *testing.T) { assert.Equal(t, pd.files[firstFile], astFileInfo) // Override - assert.NoError(t, pd.CollectAstFile(packageDir, "testdata/simple/"+firstFile.Name.String(), firstFile)) + assert.NoError(t, pd.collectAstFile(fileSet, packageDir, "testdata/simple/"+firstFile.Name.String(), firstFile)) assert.Equal(t, pd.files[firstFile], astFileInfo) // Another file secondFile := &ast.File{ Name: &ast.Ident{Name: "api.go"}, } - assert.NoError(t, pd.CollectAstFile(packageDir, "testdata/simple/"+secondFile.Name.String(), secondFile)) + assert.NoError(t, pd.collectAstFile(fileSet, packageDir, "testdata/simple/"+secondFile.Name.String(), secondFile)) } func TestPackagesDefinitions_rangeFiles(t *testing.T) { @@ -62,7 +72,7 @@ func TestPackagesDefinitions_rangeFiles(t *testing.T) { } i, expect := 0, []string{"testdata/simple/api/api.go", "testdata/simple/main.go"} - _ = rangeFiles(pd.files, func(filename string, file *ast.File) error { + _ = pd.RangeFiles(func(filename string, file *ast.File) error { assert.Equal(t, expect[i], filename) i++ return nil @@ -182,7 +192,8 @@ func TestPackagesDefinitions_FindTypeSpec(t *testing.T) { } func TestPackage_rangeFiles(t *testing.T) { - files := map[*ast.File]*AstFileInfo{ + pd := NewPackagesDefinitions() + pd.files = map[*ast.File]*AstFileInfo{ { Name: &ast.Ident{Name: "main.go"}, }: { @@ -218,10 +229,10 @@ func TestPackage_rangeFiles(t *testing.T) { sorted = append(sorted, filename) return nil } - assert.NoError(t, rangeFiles(files, processor)) + assert.NoError(t, pd.RangeFiles(processor)) assert.Equal(t, []string{"testdata/simple/api/api.go", "testdata/simple/main.go"}, sorted) - assert.Error(t, rangeFiles(files, func(filename string, file *ast.File) error { + assert.Error(t, pd.RangeFiles(func(filename string, file *ast.File) error { return ErrFuncTypeField })) diff --git a/parser.go b/parser.go index 711fe634c..1cc4546f9 100644 --- a/parser.go +++ b/parser.go @@ -211,6 +211,8 @@ func New(options ...func(*Parser)) *Parser { option(parser) } + parser.packages.debug = parser.debug + return parser } @@ -276,7 +278,6 @@ func SetDebugger(logger Debugger) func(parser *Parser) { if logger != nil { p.debug = logger } - } } @@ -377,7 +378,7 @@ func (parser *Parser) ParseAPIMultiSearchDir(searchDirs []string, mainAPIFile st return err } - err = rangeFiles(parser.packages.files, parser.ParseRouterAPIInfo) + err = parser.packages.RangeFiles(parser.ParseRouterAPIInfo) if err != nil { return err } @@ -982,7 +983,6 @@ func (parser *Parser) getTypeSchema(typeName string, file *ast.File, ref bool) ( if err == ErrRecursiveParseStruct && ref { return parser.getRefTypeSchema(typeSpecDef, schema), nil } - return nil, err } } @@ -1536,18 +1536,7 @@ func (parser *Parser) parseFile(packageDir, path string, src interface{}) error return nil } - // positions are relative to FileSet - astFile, err := goparser.ParseFile(token.NewFileSet(), path, src, goparser.ParseComments) - if err != nil { - return fmt.Errorf("ParseFile error:%+v", err) - } - - err = parser.packages.CollectAstFile(packageDir, path, astFile) - if err != nil { - return err - } - - return nil + return parser.packages.ParseFile(packageDir, path, src) } func (parser *Parser) checkOperationIDUniqueness() error { diff --git a/parser_test.go b/parser_test.go index d7735c790..c7e5bacb1 100644 --- a/parser_test.go +++ b/parser_test.go @@ -814,16 +814,13 @@ func Fun() { } }` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) + _ = p.packages.ParseFile("api", "api/api.go", src) - _, err = p.packages.ParseTypes() + _, err := p.packages.ParseTypes() assert.NoError(t, err) - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) b, _ := json.MarshalIndent(p.swagger, "", " ") @@ -2374,15 +2371,13 @@ func Test(){ } } }` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) - _, err = p.packages.ParseTypes() + _ = p.packages.ParseFile("api", "api/api.go", src) + _, err := p.packages.ParseTypes() assert.NoError(t, err) - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) out, err := json.MarshalIndent(p.swagger.Definitions, "", " ") @@ -2438,18 +2433,14 @@ type ResponseWrapper struct { }` parser := New(SetParseDependency(true)) - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - _ = parser.packages.CollectAstFile("api", "api/api.go", f) + _ = parser.packages.ParseFile("api", "api/api.go", src) - f2, err := goparser.ParseFile(token.NewFileSet(), "", restsrc, goparser.ParseComments) - assert.NoError(t, err) - _ = parser.packages.CollectAstFile("rest", "rest/rest.go", f2) + _ = parser.packages.ParseFile("rest", "rest/rest.go", restsrc) - _, err = parser.packages.ParseTypes() + _, err := parser.packages.ParseTypes() assert.NoError(t, err) - err = parser.ParseRouterAPIInfo("", f) + err = parser.packages.RangeFiles(parser.ParseRouterAPIInfo) assert.NoError(t, err) out, err := json.MarshalIndent(parser.swagger.Definitions, "", " ") @@ -2506,16 +2497,12 @@ func Test(){ } } }` - - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) - _, err = p.packages.ParseTypes() + _ = p.packages.ParseFile("api", "api/api.go", src) + _, err := p.packages.ParseTypes() assert.NoError(t, err) - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) out, err := json.MarshalIndent(p.swagger.Definitions, "", " ") @@ -2638,16 +2625,13 @@ func Test(){ } } }` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) + _ = p.packages.ParseFile("api", "api/api.go", src) - _, err = p.packages.ParseTypes() + _, err := p.packages.ParseTypes() assert.NoError(t, err) - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) out, err := json.MarshalIndent(p.swagger.Definitions, "", " ") @@ -3142,16 +3126,14 @@ func Fun() { } }` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) + err := p.packages.ParseFile("api", "api/api.go", src) + assert.NoError(t, err) _, err = p.packages.ParseTypes() assert.NoError(t, err) - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) b, _ := json.MarshalIndent(p.swagger, "", " ") @@ -3199,16 +3181,13 @@ func Fun() { } }` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) + _ = p.packages.ParseFile("api", "api/api.go", src) - _, err = p.packages.ParseTypes() + _, err := p.packages.ParseTypes() assert.NoError(t, err) - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) b, _ := json.MarshalIndent(p.swagger, "", " ") @@ -3237,15 +3216,13 @@ func Fun() { } ` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) - _, err = p.packages.ParseTypes() + _ = p.packages.ParseFile("api", "api/api.go", src) + _, err := p.packages.ParseTypes() assert.NoError(t, err) - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) assert.NoError(t, err) @@ -3282,15 +3259,12 @@ func Fun() { } } ` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) - _, err = p.packages.ParseTypes() + _ = p.packages.ParseFile("api", "api/api.go", src) + _, err := p.packages.ParseTypes() assert.NoError(t, err) - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) _, ok := p.swagger.Definitions["main.Fun.response"] @@ -3369,16 +3343,13 @@ func Fun() { } }` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) + _ = p.packages.ParseFile("api", "api/api.go", src) - _, err = p.packages.ParseTypes() + _, err := p.packages.ParseTypes() assert.NoError(t, err) - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) b, _ := json.MarshalIndent(p.swagger, "", " ") @@ -3396,16 +3367,13 @@ func Fun() { } ` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - pkgs := NewPackagesDefinitions() - // unset the .files and .packages and check that they're re-initialized by CollectAstFile + // unset the .files and .packages and check that they're re-initialized by collectAstFile pkgs.packages = nil pkgs.files = nil - _ = pkgs.CollectAstFile("api", "api/api.go", f) + _ = pkgs.ParseFile("api", "api/api.go", src) assert.NotNil(t, pkgs.packages) assert.NotNil(t, pkgs.files) } @@ -3421,18 +3389,23 @@ func Fun() { } ` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) - assert.NotNil(t, p.packages.files[f]) - - astFileInfo := p.packages.files[f] + _ = p.packages.ParseFile("api", "api/api.go", src) + assert.Equal(t, 1, len(p.packages.files)) + var path string + var file *ast.File + for path, file = range p.packages.packages["api"].Files { + break + } + assert.NotNil(t, file) + assert.NotNil(t, p.packages.files[file]) // if we collect the same again nothing should happen - _ = p.packages.CollectAstFile("api", "api/api.go", f) - assert.Equal(t, astFileInfo, p.packages.files[f]) + _ = p.packages.ParseFile("api", "api/api.go", src) + assert.Equal(t, 1, len(p.packages.files)) + assert.Equal(t, file, p.packages.packages["api"].Files[path]) + assert.NotNil(t, p.packages.files[file]) } func TestParseJSONFieldString(t *testing.T) { @@ -3560,13 +3533,11 @@ func Fun() { } ` - f, err := goparser.ParseFile(token.NewFileSet(), "", src, goparser.ParseComments) - assert.NoError(t, err) - p := New() - _ = p.packages.CollectAstFile("api", "api/api.go", f) + err := p.packages.ParseFile("api", "api/api.go", src) + assert.NoError(t, err) _, _ = p.packages.ParseTypes() - err = p.ParseRouterAPIInfo("", f) + err = p.packages.RangeFiles(p.ParseRouterAPIInfo) assert.NoError(t, err) teacher, ok := p.swagger.Definitions["Teacher"] diff --git a/types.go b/types.go index f86d8212f..79c0f6a0d 100644 --- a/types.go +++ b/types.go @@ -2,6 +2,7 @@ package swag import ( "go/ast" + "go/token" "strings" "github.com/go-openapi/spec" @@ -80,6 +81,9 @@ func (t *TypeSpecDef) FullPath() string { // AstFileInfo information of an ast.File. type AstFileInfo struct { + //FileSet the FileSet object which is used to parse this go source file + FileSet *token.FileSet + // File ast.File File *ast.File