From 30099a6add3c43706b4eec82b773b78310874935 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Sat, 20 Jan 2024 13:30:22 +0100 Subject: [PATCH] Emit nogo facts into a separate archive (#3789) * Use `-linkobj` to directly generate `.x` and `.a` * Extract nogo facts into a separate file * Address review comments --- go/private/actions/archive.bzl | 11 +- go/private/actions/compilepkg.bzl | 25 +- go/providers.rst | 10 +- go/tools/builders/BUILD.bazel | 2 - go/tools/builders/ar.go | 12 + go/tools/builders/cgo2.go | 33 +++ go/tools/builders/compilepkg.go | 82 +++---- go/tools/builders/nogo_main.go | 17 +- go/tools/builders/pack.go | 388 ------------------------------ 9 files changed, 129 insertions(+), 451 deletions(-) delete mode 100644 go/tools/builders/pack.go diff --git a/go/private/actions/archive.bzl b/go/private/actions/archive.bzl index a4e737ee68..42bf2039d7 100644 --- a/go/private/actions/archive.bzl +++ b/go/private/actions/archive.bzl @@ -58,9 +58,13 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d pre_ext += _recompile_suffix out_lib = go.declare_file(go, name = source.library.name, ext = pre_ext + ".a") - # store __.PKGDEF and nogo facts in .x + # store export information for compiling dependent packages separately out_export = go.declare_file(go, name = source.library.name, ext = pre_ext + ".x") out_cgo_export_h = None # set if cgo used in c-shared or c-archive mode + out_facts = None + nogo = go.get_nogo(go) + if nogo: + out_facts = go.declare_file(go, name = source.library.name, ext = pre_ext + ".facts") direct = [get_archive(dep) for dep in source.deps] runfiles = source.runfiles @@ -105,6 +109,8 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d archives = direct, out_lib = out_lib, out_export = out_export, + out_facts = out_facts, + nogo = nogo, out_cgo_export_h = out_cgo_export_h, gc_goopts = source.gc_goopts, cgo = True, @@ -129,6 +135,8 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d archives = direct, out_lib = out_lib, out_export = out_export, + out_facts = out_facts, + nogo = nogo, gc_goopts = source.gc_goopts, cgo = False, testfilter = testfilter, @@ -173,6 +181,7 @@ def emit_archive(go, source = None, _recompile_suffix = "", recompile_internal_d # Information needed by dependents file = out_lib, export_file = out_export, + facts_file = out_facts, data_files = as_tuple(data_files), _cgo_deps = as_tuple(cgo_deps), ) diff --git a/go/private/actions/compilepkg.bzl b/go/private/actions/compilepkg.bzl index 48adb9100d..10fa6970a2 100644 --- a/go/private/actions/compilepkg.bzl +++ b/go/private/actions/compilepkg.bzl @@ -28,6 +28,18 @@ def _archive(v): v.data.export_file.path if v.data.export_file else v.data.file.path, ) +def _facts(v): + facts_file = v.data.facts_file + if not facts_file: + return None + importpaths = [v.data.importpath] + importpaths.extend(v.data.importpath_aliases) + return "{}={}={}".format( + ":".join(importpaths), + v.data.importmap, + facts_file.path, + ) + def _embedroot_arg(src): return src.root.path @@ -55,6 +67,8 @@ def emit_compilepkg( clinkopts = [], out_lib = None, out_export = None, + out_facts = None, + nogo = None, out_cgo_export_h = None, gc_goopts = [], testfilter = None, # TODO: remove when test action compiles packages @@ -64,6 +78,8 @@ def emit_compilepkg( fail("sources is a required parameter") if out_lib == None: fail("out_lib is a required parameter") + if bool(nogo) != bool(out_facts): + fail("nogo must be specified if and only if out_facts is specified") inputs = (sources + embedsrcs + [go.package_list] + [archive.data.export_file for archive in archives] + @@ -108,10 +124,13 @@ def emit_compilepkg( args.add("-p", importmap) args.add("-package_list", go.package_list) - args.add("-o", out_lib) - args.add("-x", out_export) - nogo = go.get_nogo(go) + args.add("-lo", out_lib) + args.add("-o", out_export) if nogo: + args.add_all(archives, before_each = "-facts", map_each = _facts) + inputs.extend([archive.data.facts_file for archive in archives if archive.data.facts_file]) + args.add("-out_facts", out_facts) + outputs.append(out_facts) args.add("-nogo", nogo) inputs.append(nogo) if out_cgo_export_h: diff --git a/go/providers.rst b/go/providers.rst index dccc0e1edc..a2361ac198 100644 --- a/go/providers.rst +++ b/go/providers.rst @@ -260,7 +260,15 @@ rule. Instead, it's referenced in the ``data`` field of GoArchive_. +--------------------------------+-----------------------------------------------------------------+ | :param:`file` | :type:`File` | +--------------------------------+-----------------------------------------------------------------+ -| The archive file produced when this library is compiled. | +| The archive file for the linker produced when this library is compiled. | ++--------------------------------+-----------------------------------------------------------------+ +| :param:`export_file` | :type:`File` | ++--------------------------------+-----------------------------------------------------------------+ +| The archive file for compilation of dependent libraries produced when this library is compiled. | ++--------------------------------+-----------------------------------------------------------------+ +| :param:`facts_file` | :type:`File` | ++--------------------------------+-----------------------------------------------------------------+ +| The serialized facts for this library produced when nogo ran for this library. | +--------------------------------+-----------------------------------------------------------------+ | :param:`srcs` | :type:`tuple of File` | +--------------------------------+-----------------------------------------------------------------+ diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel index 1b44a15ca6..d327a3afbb 100644 --- a/go/tools/builders/BUILD.bazel +++ b/go/tools/builders/BUILD.bazel @@ -76,7 +76,6 @@ filegroup( "generate_test_main.go", "importcfg.go", "link.go", - "pack.go", "read.go", "replicate.go", "stdlib.go", @@ -97,7 +96,6 @@ go_source( "nogo_typeparams_go117.go", "nogo_typeparams_go118.go", "nolint.go", - "pack.go", ], # //go/tools/builders:nogo_srcs is considered a different target by # Bazel's visibility check than diff --git a/go/tools/builders/ar.go b/go/tools/builders/ar.go index 2f4b36c8b1..d2de6b96a4 100644 --- a/go/tools/builders/ar.go +++ b/go/tools/builders/ar.go @@ -23,6 +23,18 @@ import ( "strings" ) +const ( + // arHeader appears at the beginning of archives created by "ar" and + // "go tool pack" on all platforms. + arHeader = "!\n" + + // entryLength is the size in bytes of the metadata preceding each file + // in an archive. + entryLength = 60 +) + +var zeroBytes = []byte("0 ") + type header struct { NameRaw [16]byte ModTimeRaw [12]byte diff --git a/go/tools/builders/cgo2.go b/go/tools/builders/cgo2.go index fc2876a994..80043e467b 100644 --- a/go/tools/builders/cgo2.go +++ b/go/tools/builders/cgo2.go @@ -23,9 +23,11 @@ package main import ( "bytes" "fmt" + "io" "io/ioutil" "os" "path/filepath" + "runtime" "strings" ) @@ -395,3 +397,34 @@ func (e cgoError) Error() string { fmt.Fprintf(b, "Ensure that 'cgo = True' is set and the C/C++ toolchain is configured.") return b.String() } + +func copyFile(inPath, outPath string) error { + inFile, err := os.Open(inPath) + if err != nil { + return err + } + defer inFile.Close() + outFile, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + return err + } + defer outFile.Close() + _, err = io.Copy(outFile, inFile) + return err +} + +func linkFile(inPath, outPath string) error { + inPath, err := filepath.Abs(inPath) + if err != nil { + return err + } + return os.Symlink(inPath, outPath) +} + +func copyOrLinkFile(inPath, outPath string) error { + if runtime.GOOS == "windows" { + return copyFile(inPath, outPath) + } else { + return linkFile(inPath, outPath) + } +} diff --git a/go/tools/builders/compilepkg.go b/go/tools/builders/compilepkg.go index 81800fed69..360f69fe0d 100644 --- a/go/tools/builders/compilepkg.go +++ b/go/tools/builders/compilepkg.go @@ -50,9 +50,9 @@ func compilePkg(args []string) error { fs := flag.NewFlagSet("GoCompilePkg", flag.ExitOnError) goenv := envFlags(fs) var unfilteredSrcs, coverSrcs, embedSrcs, embedLookupDirs, embedRoots, recompileInternalDeps multiFlag - var deps archiveMultiFlag + var deps, facts archiveMultiFlag var importPath, packagePath, nogoPath, packageListPath, coverMode string - var outPath, outFactsPath, cgoExportHPath string + var outLinkobjPath, outInterfacePath, outFactsPath, cgoExportHPath string var testFilter string var gcFlags, asmFlags, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags quoteMultiFlag var coverFormat string @@ -63,6 +63,7 @@ func compilePkg(args []string) error { fs.Var(&embedLookupDirs, "embedlookupdir", "Root-relative paths to directories relative to which //go:embed directives are resolved") fs.Var(&embedRoots, "embedroot", "Bazel output root under which a file passed via -embedsrc resides") fs.Var(&deps, "arc", "Import path, package path, and file name of a direct dependency, separated by '='") + fs.Var(&facts, "facts", "Import path, package path, and file name of a direct dependency's nogo facts file, separated by '='") fs.StringVar(&importPath, "importpath", "", "The import path of the package being compiled. Not passed to the compiler, but may be displayed in debug data.") fs.StringVar(&packagePath, "p", "", "The package path (importmap) of the package being compiled") fs.Var(&gcFlags, "gcflags", "Go compiler flags") @@ -76,8 +77,9 @@ func compilePkg(args []string) error { fs.StringVar(&nogoPath, "nogo", "", "The nogo binary. If unset, nogo will not be run.") fs.StringVar(&packageListPath, "package_list", "", "The file containing the list of standard library packages") fs.StringVar(&coverMode, "cover_mode", "", "The coverage mode to use. Empty if coverage instrumentation should not be added.") - fs.StringVar(&outPath, "o", "", "The output archive file to write compiled code") - fs.StringVar(&outFactsPath, "x", "", "The output archive file to write export data and nogo facts") + fs.StringVar(&outLinkobjPath, "lo", "", "The full output archive file required by the linker") + fs.StringVar(&outInterfacePath, "o", "", "The export-only output archive required to compile dependent packages") + fs.StringVar(&outFactsPath, "out_facts", "", "The file to emit serialized nogo facts to (must be set if -nogo is set") fs.StringVar(&cgoExportHPath, "cgoexport", "", "The _cgo_exports.h file to write") fs.StringVar(&testFilter, "testfilter", "off", "Controls test package filtering") fs.StringVar(&coverFormat, "cover_format", "", "Emit source file paths in coverage instrumentation suitable for the specified coverage format") @@ -94,7 +96,7 @@ func compilePkg(args []string) error { } cgoEnabled := os.Getenv("CGO_ENABLED") == "1" cc := os.Getenv("CC") - outPath = abs(outPath) + outLinkobjPath = abs(outLinkobjPath) for i := range unfilteredSrcs { unfilteredSrcs[i] = abs(unfilteredSrcs[i]) } @@ -142,6 +144,7 @@ func compilePkg(args []string) error { packagePath, srcs, deps, + facts, coverMode, coverSrcs, embedSrcs, @@ -159,7 +162,8 @@ func compilePkg(args []string) error { ldFlags, nogoPath, packageListPath, - outPath, + outLinkobjPath, + outInterfacePath, outFactsPath, cgoExportHPath, coverFormat, @@ -173,6 +177,7 @@ func compileArchive( packagePath string, srcs archiveSrcs, deps []archive, + facts []archive, coverMode string, coverSrcs []string, embedSrcs []string, @@ -190,8 +195,9 @@ func compileArchive( ldFlags []string, nogoPath string, packageListPath string, - outPath string, - outXPath string, + outLinkObj string, + outInterfacePath string, + outFactsPath string, cgoExportHPath string, coverFormat string, recompileInternalDeps []string, @@ -209,7 +215,7 @@ func compileArchive( // Otherwise, GoPack will complain if we try to add assembly or cgo objects. // A truly empty archive does not include any references to source file paths, which // ensures hermeticity even though the temp file path is random. - emptyGoFile, err := os.CreateTemp(filepath.Dir(outPath), "*.go") + emptyGoFile, err := os.CreateTemp(filepath.Dir(outLinkObj), "*.go") if err != nil { return err } @@ -400,7 +406,7 @@ func compileArchive( } // Build an importcfg file for the compiler. - importcfgPath, err := buildImportcfgFileForCompile(imports, goenv.installSuffix, filepath.Dir(outPath)) + importcfgPath, err := buildImportcfgFileForCompile(imports, goenv.installSuffix, filepath.Dir(outLinkObj)) if err != nil { return err } @@ -443,12 +449,11 @@ func compileArchive( // Run nogo concurrently. var nogoChan chan error - outFactsPath := filepath.Join(workDir, nogoFact) - if nogoPath != "" && len(goSrcsNogo) > 0 { + if nogoPath != "" { ctx, cancel := context.WithCancel(context.Background()) nogoChan = make(chan error) go func() { - nogoChan <- runNogo(ctx, workDir, nogoPath, goSrcsNogo, deps, packagePath, importcfgPath, outFactsPath) + nogoChan <- runNogo(ctx, workDir, nogoPath, goSrcsNogo, facts, packagePath, importcfgPath, outFactsPath) }() defer func() { if nogoChan != nil { @@ -478,7 +483,7 @@ func compileArchive( } // Compile the filtered .go files. - if err := compileGo(goenv, goSrcs, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath, gcFlags, pgoprofile, outPath); err != nil { + if err := compileGo(goenv, goSrcs, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath, gcFlags, pgoprofile, outLinkObj, outInterfacePath); err != nil { return err } @@ -512,44 +517,25 @@ func compileArchive( // Pack .o files into the archive. These may come from cgo generated code, // cgo dependencies (cdeps), or assembly. if len(objFiles) > 0 { - if err := appendFiles(goenv, outPath, objFiles); err != nil { + if err := appendToArchive(goenv, outLinkObj, objFiles); err != nil { return err } } // Check results from nogo. - nogoStatus := nogoNotRun if nogoChan != nil { err := <-nogoChan nogoChan = nil // no cancellation needed if err != nil { - nogoStatus = nogoFailed - // TODO: should we still create the .x file without nogo facts in this case? + // TODO: Move nogo into a separate action so we don't fail the compilation here. return err } - nogoStatus = nogoSucceeded - } - - // Extract the export data file and pack it in an .x archive together with the - // nogo facts file (if there is one). This allows compile actions to depend - // on .x files only, so we don't need to recompile a package when one of its - // imports changes in a way that doesn't affect export data. - // TODO(golang/go#33820): After Go 1.16 is the minimum supported version, - // use -linkobj to tell the compiler to create separate .a and .x files for - // compiled code and export data. Before that version, the linker needed - // export data in the .a file when building a plugin. To work around that, - // we copy the export data into .x ourselves. - if err = extractFileFromArchive(outPath, workDir, pkgDef); err != nil { - return err - } - pkgDefPath := filepath.Join(workDir, pkgDef) - if nogoStatus == nogoSucceeded { - return appendFiles(goenv, outXPath, []string{pkgDefPath, outFactsPath}) } - return appendFiles(goenv, outXPath, []string{pkgDefPath}) + + return nil } -func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath string, gcFlags []string, pgoprofile string, outPath string) error { +func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath string, gcFlags []string, pgoprofile, outLinkobjPath, outInterfacePath string) error { args := goenv.goTool("compile") args = append(args, "-p", packagePath, "-importcfg", importcfgPath, "-pack") if embedcfgPath != "" { @@ -565,19 +551,24 @@ func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPa args = append(args, "-pgoprofile", pgoprofile) } args = append(args, gcFlags...) - args = append(args, "-o", outPath) + args = append(args, "-o", outInterfacePath) + args = append(args, "-linkobj", outLinkobjPath) args = append(args, "--") args = append(args, srcs...) absArgs(args, []string{"-I", "-o", "-trimpath", "-importcfg"}) return goenv.runCommand(args) } -func runNogo(ctx context.Context, workDir string, nogoPath string, srcs []string, deps []archive, packagePath, importcfgPath, outFactsPath string) error { +func runNogo(ctx context.Context, workDir string, nogoPath string, srcs []string, facts []archive, packagePath, importcfgPath, outFactsPath string) error { + if len(srcs) == 0 { + // emit_compilepkg expects a nogo facts file, even if it's empty. + return os.WriteFile(outFactsPath, nil, 0o666) + } args := []string{nogoPath} args = append(args, "-p", packagePath) args = append(args, "-importcfg", importcfgPath) - for _, dep := range deps { - args = append(args, "-fact", fmt.Sprintf("%s=%s", dep.importPath, dep.file)) + for _, fact := range facts { + args = append(args, "-fact", fmt.Sprintf("%s=%s", fact.importPath, fact.file)) } args = append(args, "-x", outFactsPath) args = append(args, srcs...) @@ -607,6 +598,13 @@ func runNogo(ctx context.Context, workDir string, nogoPath string, srcs []string return nil } +func appendToArchive(goenv *env, outPath string, objFiles []string) error { + // Use abs to work around long path issues on Windows. + args := goenv.goTool("pack", "r", abs(outPath)) + args = append(args, objFiles...) + return goenv.runCommand(args) +} + func createTrimPath(gcFlags []string, path string) string { for _, flag := range gcFlags { if strings.HasPrefix(flag, "-trimpath=") { diff --git a/go/tools/builders/nogo_main.go b/go/tools/builders/nogo_main.go index 17ff5314c7..23acdef0eb 100644 --- a/go/tools/builders/nogo_main.go +++ b/go/tools/builders/nogo_main.go @@ -610,8 +610,8 @@ func (i *importer) Import(path string) (*types.Package, error) { } func (i *importer) readFacts(pkgPath string) ([]byte, error) { - archive := i.factMap[pkgPath] - if archive == "" { + facts := i.factMap[pkgPath] + if facts == "" { // Packages that were not built with the nogo toolchain will not be // analyzed, so there's no opportunity to store facts. This includes // packages in the standard library and packages built with go_tool_library, @@ -621,18 +621,7 @@ func (i *importer) readFacts(pkgPath string) ([]byte, error) { // fmt.Printf accepts a format string. return nil, nil } - factReader, err := readFileInArchive(nogoFact, archive) - if os.IsNotExist(err) { - // Packages that were not built with the nogo toolchain will not be - // analyzed, so there's no opportunity to store facts. This includes - // packages in the standard library and packages built with go_tool_library, - // such as coverdata. - return nil, nil - } else if err != nil { - return nil, err - } - defer factReader.Close() - return ioutil.ReadAll(factReader) + return os.ReadFile(facts) } type factMultiFlag map[string]string diff --git a/go/tools/builders/pack.go b/go/tools/builders/pack.go deleted file mode 100644 index ddbb1930b3..0000000000 --- a/go/tools/builders/pack.go +++ /dev/null @@ -1,388 +0,0 @@ -// Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" -) - -func copyFile(inPath, outPath string) error { - inFile, err := os.Open(inPath) - if err != nil { - return err - } - defer inFile.Close() - outFile, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) - if err != nil { - return err - } - defer outFile.Close() - _, err = io.Copy(outFile, inFile) - return err -} - -func linkFile(inPath, outPath string) error { - inPath, err := filepath.Abs(inPath) - if err != nil { - return err - } - return os.Symlink(inPath, outPath) -} - -func copyOrLinkFile(inPath, outPath string) error { - if runtime.GOOS == "windows" { - return copyFile(inPath, outPath) - } else { - return linkFile(inPath, outPath) - } -} - -const ( - // arHeader appears at the beginning of archives created by "ar" and - // "go tool pack" on all platforms. - arHeader = "!\n" - - // entryLength is the size in bytes of the metadata preceding each file - // in an archive. - entryLength = 60 - - // pkgDef is the name of the export data file within an archive - pkgDef = "__.PKGDEF" - - // nogoFact is the name of the nogo fact file - nogoFact = "nogo.out" -) - -var zeroBytes = []byte("0 ") - -type bufioReaderWithCloser struct { - // bufio.Reader is needed to skip bytes in archives - *bufio.Reader - io.Closer -} - -func extractFiles(archive, dir string, names map[string]struct{}) (files []string, err error) { - rc, err := openArchive(archive) - if err != nil { - return nil, err - } - defer rc.Close() - - var nameData []byte - bufReader := rc.Reader - for { - name, size, err := readMetadata(bufReader, &nameData) - if err == io.EOF { - return files, nil - } - if err != nil { - return nil, err - } - if !isObjectFile(name) { - if err := skipFile(bufReader, size); err != nil { - return nil, err - } - continue - } - name, err = simpleName(name, names) - if err != nil { - return nil, err - } - name = filepath.Join(dir, name) - if err := extractFile(bufReader, name, size); err != nil { - return nil, err - } - files = append(files, name) - } -} - -func openArchive(archive string) (bufioReaderWithCloser, error) { - f, err := os.Open(archive) - if err != nil { - return bufioReaderWithCloser{}, err - } - r := bufio.NewReader(f) - header := make([]byte, len(arHeader)) - if _, err := io.ReadFull(r, header); err != nil || string(header) != arHeader { - f.Close() - return bufioReaderWithCloser{}, fmt.Errorf("%s: bad header", archive) - } - return bufioReaderWithCloser{r, f}, nil -} - -// readMetadata reads the relevant fields of an entry. Before calling, -// r must be positioned at the beginning of an entry. Afterward, r will -// be positioned at the beginning of the file data. io.EOF is returned if -// there are no more files in the archive. -// -// Both BSD and GNU / SysV naming conventions are supported. -func readMetadata(r *bufio.Reader, nameData *[]byte) (name string, size int64, err error) { -retry: - // Each file is preceded by a 60-byte header that contains its metadata. - // We only care about two fields, name and size. Other fields (mtime, - // owner, group, mode) are ignored because they don't affect compilation. - var entry [entryLength]byte - if _, err := io.ReadFull(r, entry[:]); err != nil { - return "", 0, err - } - - sizeField := strings.TrimSpace(string(entry[48:58])) - size, err = strconv.ParseInt(sizeField, 10, 64) - if err != nil { - return "", 0, err - } - - nameField := strings.TrimRight(string(entry[:16]), " ") - switch { - case strings.HasPrefix(nameField, "#1/"): - // BSD-style name. The number of bytes in the name is written here in - // ASCII, right-padded with spaces. The actual name is stored at the - // beginning of the file data, left-padded with NUL bytes. - nameField = nameField[len("#1/"):] - nameLen, err := strconv.ParseInt(nameField, 10, 64) - if err != nil { - return "", 0, err - } - nameBuf := make([]byte, nameLen) - if _, err := io.ReadFull(r, nameBuf); err != nil { - return "", 0, err - } - name = strings.TrimRight(string(nameBuf), "\x00") - size -= nameLen - - case nameField == "//": - // GNU / SysV-style name data. This is a fake file that contains names - // for files with long names. We read this into nameData, then read - // the next entry. - *nameData = make([]byte, size) - if _, err := io.ReadFull(r, *nameData); err != nil { - return "", 0, err - } - if size%2 != 0 { - // Files are aligned at 2-byte offsets. Discard the padding byte if the - // size was odd. - if _, err := r.ReadByte(); err != nil { - return "", 0, err - } - } - goto retry - - case nameField == "/": - // GNU / SysV-style symbol lookup table. Skip. - if err := skipFile(r, size); err != nil { - return "", 0, err - } - goto retry - - case strings.HasPrefix(nameField, "/"): - // GNU / SysV-style long file name. The number that follows the slash is - // an offset into the name data that should have been read earlier. - // The file name ends with a slash. - nameField = nameField[1:] - nameOffset, err := strconv.Atoi(nameField) - if err != nil { - return "", 0, err - } - if nameData == nil || nameOffset < 0 || nameOffset >= len(*nameData) { - return "", 0, fmt.Errorf("invalid name length: %d", nameOffset) - } - i := bytes.IndexByte((*nameData)[nameOffset:], '/') - if i < 0 { - return "", 0, errors.New("file name does not end with '/'") - } - name = string((*nameData)[nameOffset : nameOffset+i]) - - case strings.HasSuffix(nameField, "/"): - // GNU / SysV-style short file name. - name = nameField[:len(nameField)-1] - - default: - // Common format name. - name = nameField - } - - return name, size, err -} - -// extractFile reads size bytes from r and writes them to a new file, name. -func extractFile(r *bufio.Reader, name string, size int64) error { - w, err := os.Create(name) - if err != nil { - return err - } - defer w.Close() - _, err = io.CopyN(w, r, size) - if err != nil { - return err - } - if size%2 != 0 { - // Files are aligned at 2-byte offsets. Discard the padding byte if the - // size was odd. - if _, err := r.ReadByte(); err != nil { - return err - } - } - return nil -} - -func skipFile(r *bufio.Reader, size int64) error { - if size%2 != 0 { - // Files are aligned at 2-byte offsets. Discard the padding byte if the - // size was odd. - size += 1 - } - _, err := r.Discard(int(size)) - return err -} - -func isObjectFile(name string) bool { - return strings.HasSuffix(name, ".o") -} - -// simpleName returns a file name which is at most 15 characters -// and doesn't conflict with other names. If it is not possible to choose -// such a name, simpleName will truncate the given name to 15 characters. -// The original file extension will be preserved. -func simpleName(name string, names map[string]struct{}) (string, error) { - if _, ok := names[name]; !ok && len(name) < 16 { - names[name] = struct{}{} - return name, nil - } - var stem, ext string - if i := strings.LastIndexByte(name, '.'); i < 0 { - stem = name - } else { - stem = strings.Replace(name[:i], ".", "_", -1) - ext = name[i:] - } - for n := 0; n < len(names)+1; n++ { - ns := strconv.Itoa(n) - stemLen := 15 - len(ext) - len(ns) - if stemLen < 0 { - break - } - if stemLen > len(stem) { - stemLen = len(stem) - } - candidate := stem[:stemLen] + ns + ext - if _, ok := names[candidate]; !ok { - names[candidate] = struct{}{} - return candidate, nil - } - } - return "", fmt.Errorf("cannot shorten file name: %q", name) -} - -func appendFiles(goenv *env, archive string, files []string) error { - archive = abs(archive) // required for long filenames on Windows. - - // Create an empty archive if one doesn't already exist. - // In Go 1.16, 'go tool pack r' reports an error if the archive doesn't exist. - // 'go tool pack c' copies export data in addition to creating the archive, - // so we don't want to use that directly. - _, err := os.Stat(archive) - if err != nil && !os.IsNotExist(err) { - return err - } - if os.IsNotExist(err) { - if err := ioutil.WriteFile(archive, []byte(arHeader), 0666); err != nil { - return err - } - } - - // Append files to the archive. - // TODO(jayconrod): copy cmd/internal/archive and use that instead of - // shelling out to cmd/pack. - args := goenv.goTool("pack", "r", archive) - args = append(args, files...) - return goenv.runCommand(args) -} - -type readWithCloser struct { - io.Reader - io.Closer -} - -func readFileInArchive(fileName, archive string) (io.ReadCloser, error) { - rc, err := openArchive(archive) - if err != nil { - return nil, err - } - var nameData []byte - bufReader := rc.Reader - for err == nil { - // avoid shadowing err in the loop it can be returned correctly in the end - var ( - name string - size int64 - ) - name, size, err = readMetadata(bufReader, &nameData) - if err != nil { - break - } - if name == fileName { - return readWithCloser{ - Reader: io.LimitReader(rc, size), - Closer: rc, - }, nil - } - err = skipFile(bufReader, size) - } - if err == io.EOF { - err = os.ErrNotExist - } - rc.Close() - return nil, err -} - -func extractFileFromArchive(archive, dir, name string) (err error) { - archiveReader, err := readFileInArchive(name, archive) - if err != nil { - return fmt.Errorf("error reading %s from %s: %v", name, archive, err) - } - defer func() { - e := archiveReader.Close() - if e != nil && err == nil { - err = fmt.Errorf("error closing %q: %v", archive, e) - } - }() - outPath := filepath.Join(dir, pkgDef) - outFile, err := os.Create(outPath) - if err != nil { - return fmt.Errorf("error creating %s: %v", outPath, err) - } - defer func() { - e := outFile.Close() - if e != nil && err == nil { - err = fmt.Errorf("error closing %q: %v", outPath, e) - } - }() - if size, err := io.Copy(outFile, archiveReader); err != nil { - return fmt.Errorf("error writing %s: %v", outPath, err) - } else if size == 0 { - return fmt.Errorf("%s is empty in %s", name, archive) - } - return err -}