Skip to content

Commit

Permalink
stdliblist: return generated files (#3552)
Browse files Browse the repository at this point in the history 
The stdlib json currently is not populated with generated files. When
CGO is being used, this means that the file list returned does not
actually compile. This change adds `-compiled=true` to go list's output
which results in the full set required to compile stdlib.

Fixes #3523

Change-Id: Ic06243e7a5de68946d87ff9734156c522bfd8662

Co-authored-by: Thomas Rampelberg <[email protected]>
  • Loading branch information
@grampelberg
grampelberg and Thomas Rampelberg authored May 10, 2023
1 parent c188072 commit c403db6
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 33 deletions.
57 changes: 35 additions & 22 deletions go/private/actions/stdlib.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,16 +68,46 @@ def _build_stdlib_list_json(go):
args = go.builder_args(go, "stdliblist")
args.add("-sdk", go.sdk.root_file.dirname)
args.add("-out", out)

inputs = go.sdk_files
if not go.mode.pure:
inputs += go.crosstool

go.actions.run(
inputs = go.sdk_files,
inputs = inputs,
outputs = [out],
mnemonic = "GoStdlibList",
executable = go.toolchain._builder,
arguments = [args],
env = go.env,
env = _build_env(go),
)
return out

def _build_env(go):
env = go.env

if go.mode.pure:
env.update({"CGO_ENABLED": "0"})
return env

# NOTE(#2545): avoid unnecessary dynamic link
# go std library doesn't use C++, so should not have -lstdc++
# Also drop coverage flags as nothing in the stdlib is compiled with
# coverage - we disable it for all CGo code anyway.
ldflags = [
option
for option in extldflags_from_cc_toolchain(go)
if option not in ("-lstdc++", "-lc++") and option not in COVERAGE_OPTIONS_DENYLIST
]
env.update({
"CGO_ENABLED": "1",
"CC": go.cgo_tools.c_compiler_path,
"CGO_CFLAGS": " ".join(go.cgo_tools.c_compile_options),
"CGO_LDFLAGS": " ".join(ldflags),
})

return env

def _sdk_stdlib(go):
return GoStdLib(
_list_json = _build_stdlib_list_json(go),
Expand All @@ -96,26 +126,9 @@ def _build_stdlib(go):
if not go.mode.pure:
args.add("-package", "runtime/cgo")
args.add_all(link_mode_args(go.mode))
env = go.env
if go.mode.pure:
env.update({"CGO_ENABLED": "0"})
else:
# NOTE(#2545): avoid unnecessary dynamic link
# go std library doesn't use C++, so should not have -lstdc++
# Also drop coverage flags as nothing in the stdlib is compiled with
# coverage - we disable it for all CGo code anyway.
ldflags = [
option
for option in extldflags_from_cc_toolchain(go)
if option not in ("-lstdc++", "-lc++") and option not in COVERAGE_OPTIONS_DENYLIST
]
env.update({
"CGO_ENABLED": "1",
"CC": go.cgo_tools.c_compiler_path,
"CGO_CFLAGS": " ".join(go.cgo_tools.c_compile_options),
"CGO_LDFLAGS": " ".join(ldflags),
})

args.add("-gcflags", quote_opts(go.mode.gc_goopts))

inputs = (go.sdk.srcs +
go.sdk.headers +
go.sdk.tools +
Expand All @@ -128,7 +141,7 @@ def _build_stdlib(go):
mnemonic = "GoStdlib",
executable = go.toolchain._builder,
arguments = [args],
env = env,
env = _build_env(go),
)
return GoStdLib(
_list_json = _build_stdlib_list_json(go),
Expand Down
73 changes: 62 additions & 11 deletions go/tools/builders/stdliblist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,20 +116,43 @@ func outputBasePath(cloneBase, p string) string {
return filepath.Join("__BAZEL_OUTPUT_BASE__", dir)
}

// absoluteSourcesPaths replace cloneBase of the absolution
// paths with the label for all source files in a package
// absoluteSourcesPaths replace cloneBase of the absolution
// paths with the label for all source files in a package
func absoluteSourcesPaths(cloneBase, pkgDir string, srcs []string) []string {
ret := make([]string, 0, len(srcs))
pkgDir = outputBasePath(cloneBase, pkgDir)
for _, src := range srcs {
ret = append(ret, filepath.Join(pkgDir, src))
absPath := src

// Generated files will already have an absolute path. These come from
// the compiler's cache.
if !filepath.IsAbs(src) {
absPath = filepath.Join(pkgDir, src)
}

ret = append(ret, absPath)
}
return ret
}

// filterGoFiles keeps only files either ending in .go or those without an
// extension (which are from the cache). This is a work around for
// https://golang.org/issue/28749: cmd/go puts assembly, C, and C++ files in
// CompiledGoFiles.
func filterGoFiles(srcs []string) []string {
ret := make([]string, 0, len(srcs))
for _, f := range srcs {
if ext := filepath.Ext(f); ext == ".go" || ext == "" {
ret = append(ret, f)
}
}

return ret
}

func flatPackageForStd(cloneBase string, pkg *goListPackage) *flatPackage {
// Don't use generated files from the stdlib
goFiles := absoluteSourcesPaths(cloneBase, pkg.Dir, pkg.GoFiles)
compiledGoFiles := absoluteSourcesPaths(cloneBase, pkg.Dir, pkg.CompiledGoFiles)

newPkg := &flatPackage{
ID: stdlibPackageID(pkg.ImportPath),
Expand All @@ -139,13 +162,29 @@ func flatPackageForStd(cloneBase string, pkg *goListPackage) *flatPackage {
Imports: map[string]string{},
Standard: pkg.Standard,
GoFiles: goFiles,
CompiledGoFiles: goFiles,
CompiledGoFiles: filterGoFiles(compiledGoFiles),
}

// imports
//
// Imports contains the IDs of all imported packages.
// ImportsMap records (path, ID) only where they differ.
ids := make(map[string]struct{})
for _, id := range pkg.Imports {
ids[id] = struct{}{}
}

for path, id := range pkg.ImportMap {
newPkg.Imports[path] = stdlibPackageID(id)
delete(ids, id)
}
for _, imp := range pkg.Imports {
newPkg.Imports[imp] = stdlibPackageID(imp)

for id := range ids {
if id != "C" {
newPkg.Imports[id] = stdlibPackageID(id)
}
}
// We don't support CGo for now
delete(newPkg.Imports, "C")

return newPkg
}

Expand Down Expand Up @@ -200,12 +239,18 @@ func stdliblist(args []string) error {
}
os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator)))
os.Setenv("GOROOT", newGoRoot)

cgoEnabled := os.Getenv("CGO_ENABLED") == "1"
// Make sure we have an absolute path to the C compiler.
// TODO(#1357): also take absolute paths of includes and other paths in flags.
os.Setenv("CC", quotePathIfNeeded(abs(os.Getenv("CC"))))
ccEnv, ok := os.LookupEnv("CC")
if cgoEnabled && !ok {
return fmt.Errorf("CC must be set")
}
os.Setenv("CC", quotePathIfNeeded(abs(ccEnv)))

// We want to keep the cache around so that the processed files can be used by other tools.
cachePath := abs(*out + ".gocache")
defer os.RemoveAll(cachePath)
os.Setenv("GOCACHE", cachePath)
os.Setenv("GOMODCACHE", cachePath)
os.Setenv("GOPATH", cachePath)
Expand All @@ -214,6 +259,11 @@ func stdliblist(args []string) error {
if len(build.Default.BuildTags) > 0 {
listArgs = append(listArgs, "-tags", strings.Join(build.Default.BuildTags, ","))
}

if cgoEnabled {
listArgs = append(listArgs, "-compiled=true")
}

listArgs = append(listArgs, "-json", "builtin", "std", "runtime/cgo")

jsonFile, err := os.Create(*out)
Expand All @@ -226,6 +276,7 @@ func stdliblist(args []string) error {
if err := goenv.runCommandToFile(jsonData, os.Stderr, listArgs); err != nil {
return err
}

encoder := json.NewEncoder(jsonFile)
decoder := json.NewDecoder(jsonData)
for decoder.More() {
Expand Down

0 comments on commit c403db6

Please sign in to comment.