Skip to content

Commit 8028f3f

Browse files
committed
wip: stdlib
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
1 parent 87d7f4c commit 8028f3f

7 files changed

Lines changed: 181 additions & 20 deletions

File tree

syft/pkg/cataloger/golang/cataloger.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,11 @@ func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
2626

2727
// NewGoModuleBinaryCataloger returns a new cataloger object that searches within binaries built by the go compiler.
2828
func NewGoModuleBinaryCataloger(opts CatalogerConfig) pkg.Cataloger {
29+
c := newGoBinaryCataloger(opts)
2930
return generic.NewCataloger(binaryCatalogerName).
3031
WithParserByMimeTypes(
31-
newGoBinaryCataloger(opts).parseGoBinary,
32+
c.parseGoBinary,
3233
mimetype.ExecutableMIMETypeSet.List()...,
3334
).
34-
WithResolvingProcessors(stdlibProcessor)
35+
WithResolvingProcessors(c.stdlibProcessor)
3536
}

syft/pkg/cataloger/golang/config_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ func Test_Config(t *testing.T) {
5858
NoProxy: []string{"my.private", "no.proxy"},
5959
MainModuleVersion: DefaultMainModuleVersionConfig(),
6060
UsePackagesLib: true,
61+
CaptureSymbols: true,
6162
},
6263
},
6364
{
@@ -86,6 +87,7 @@ func Test_Config(t *testing.T) {
8687
NoProxy: []string{"alt.no.proxy"},
8788
MainModuleVersion: DefaultMainModuleVersionConfig(),
8889
UsePackagesLib: true,
90+
CaptureSymbols: true,
8991
},
9092
},
9193
}

syft/pkg/cataloger/golang/parse_go_binary.go

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"runtime/debug"
1414
"slices"
1515
"strings"
16+
"sync"
1617
"time"
1718

1819
"golang.org/x/mod/module"
@@ -50,14 +51,41 @@ type goBinaryCataloger struct {
5051
licenseResolver goLicenseResolver
5152
mainModuleVersion MainModuleVersionConfig
5253
captureSymbols bool
54+
55+
// stdlibSymbols holds the standard-library function symbols discovered per binary (keyed by the
56+
// binary's location), populated during parsing and consumed by stdlibProcessor when it builds the
57+
// synthetic "stdlib" package. Guarded by stdlibSymbolsMu because parsers run concurrently.
58+
stdlibSymbols map[file.Coordinates][]string
59+
stdlibSymbolsMu sync.Mutex
5360
}
5461

5562
func newGoBinaryCataloger(opts CatalogerConfig) *goBinaryCataloger {
5663
return &goBinaryCataloger{
5764
licenseResolver: newGoLicenseResolver(binaryCatalogerName, opts),
5865
mainModuleVersion: opts.MainModuleVersion,
5966
captureSymbols: opts.CaptureSymbols,
67+
stdlibSymbols: make(map[file.Coordinates][]string),
68+
}
69+
}
70+
71+
// recordStdlibSymbols merges the standard-library symbols discovered for a binary location so the
72+
// stdlib processor can attach them to the synthetic stdlib package.
73+
func (c *goBinaryCataloger) recordStdlibSymbols(coord file.Coordinates, symbols []string) {
74+
if len(symbols) == 0 {
75+
return
6076
}
77+
c.stdlibSymbolsMu.Lock()
78+
defer c.stdlibSymbolsMu.Unlock()
79+
merged := append(c.stdlibSymbols[coord], symbols...)
80+
slices.Sort(merged)
81+
c.stdlibSymbols[coord] = slices.Compact(merged)
82+
}
83+
84+
// stdlibSymbolsFor returns the standard-library symbols recorded for a binary location.
85+
func (c *goBinaryCataloger) stdlibSymbolsFor(coord file.Coordinates) []string {
86+
c.stdlibSymbolsMu.Lock()
87+
defer c.stdlibSymbolsMu.Unlock()
88+
return c.stdlibSymbols[coord]
6189
}
6290

6391
// parseGoBinary catalogs packages found in the "buildinfo" section of a binary built by the go compiler.
@@ -130,7 +158,8 @@ func (c *goBinaryCataloger) buildGoPkgInfo(ctx context.Context, resolver file.Re
130158
mod.Main = createMainModuleFromPath(mod)
131159
}
132160

133-
symbolsByModule := moduleSymbols(mod.symbols, &mod.Main, mod.Deps)
161+
symbolsByModule, stdlibSymbols := moduleSymbols(mod.symbols, &mod.Main, mod.Deps)
162+
c.recordStdlibSymbols(location.Coordinates, stdlibSymbols)
134163

135164
var pkgs []pkg.Package
136165
for _, dep := range mod.Deps {

syft/pkg/cataloger/golang/stdlib_package.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,12 @@ import (
1212
"github.com/anchore/syft/syft/pkg"
1313
)
1414

15-
func stdlibProcessor(ctx context.Context, _ file.Resolver, pkgs []pkg.Package, relationships []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
16-
compilerPkgs, newRelationships := stdlibPackageAndRelationships(ctx, pkgs)
15+
func (c *goBinaryCataloger) stdlibProcessor(ctx context.Context, _ file.Resolver, pkgs []pkg.Package, relationships []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
16+
compilerPkgs, newRelationships := c.stdlibPackageAndRelationships(ctx, pkgs)
1717
return append(pkgs, compilerPkgs...), append(relationships, newRelationships...), err
1818
}
1919

20-
func stdlibPackageAndRelationships(ctx context.Context, pkgs []pkg.Package) ([]pkg.Package, []artifact.Relationship) {
20+
func (c *goBinaryCataloger) stdlibPackageAndRelationships(ctx context.Context, pkgs []pkg.Package) ([]pkg.Package, []artifact.Relationship) {
2121
var goCompilerPkgs []pkg.Package
2222
var relationships []artifact.Relationship
2323
totalLocations := file.NewLocationSet()
@@ -33,7 +33,7 @@ func stdlibPackageAndRelationships(ctx context.Context, pkgs []pkg.Package) ([]p
3333
continue
3434
}
3535

36-
stdLibPkg := newGoStdLib(ctx, mValue.GoCompiledVersion, goPkg.Locations)
36+
stdLibPkg := newGoStdLib(ctx, mValue.GoCompiledVersion, goPkg.Locations, c.stdlibSymbolsFor(location.Coordinates))
3737
if stdLibPkg == nil {
3838
continue
3939
}
@@ -50,7 +50,7 @@ func stdlibPackageAndRelationships(ctx context.Context, pkgs []pkg.Package) ([]p
5050
return goCompilerPkgs, relationships
5151
}
5252

53-
func newGoStdLib(ctx context.Context, version string, location file.LocationSet) *pkg.Package {
53+
func newGoStdLib(ctx context.Context, version string, location file.LocationSet, symbols []string) *pkg.Package {
5454
stdlibCpe, err := generateStdlibCpe(version)
5555
if err != nil {
5656
return nil
@@ -66,6 +66,7 @@ func newGoStdLib(ctx context.Context, version string, location file.LocationSet)
6666
Type: pkg.GoModulePkg,
6767
Metadata: pkg.GolangBinaryBuildinfoEntry{
6868
GoCompiledVersion: version,
69+
Symbols: symbols,
6970
},
7071
}
7172
goCompilerPkg.SetID()

syft/pkg/cataloger/golang/stdlib_package_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ func Test_stdlibPackageAndRelationships(t *testing.T) {
8888
}
8989
for _, tt := range tests {
9090
t.Run(tt.name, func(t *testing.T) {
91-
gotPkgs, gotRels := stdlibPackageAndRelationships(ctx, tt.pkgs)
91+
c := &goBinaryCataloger{stdlibSymbols: make(map[file.Coordinates][]string)}
92+
gotPkgs, gotRels := c.stdlibPackageAndRelationships(ctx, tt.pkgs)
9293
assert.Len(t, gotPkgs, tt.wantPkgs)
9394
assert.Len(t, gotRels, tt.wantRels)
9495
})
@@ -137,7 +138,8 @@ func Test_stdlibPackageAndRelationships_values(t *testing.T) {
137138
Type: artifact.DependencyOfRelationship,
138139
}
139140

140-
gotPkgs, gotRels := stdlibPackageAndRelationships(ctx, []pkg.Package{p})
141+
c := &goBinaryCataloger{stdlibSymbols: make(map[file.Coordinates][]string)}
142+
gotPkgs, gotRels := c.stdlibPackageAndRelationships(ctx, []pkg.Package{p})
141143
require.Len(t, gotPkgs, 1)
142144

143145
gotPkg := gotPkgs[0]

syft/pkg/cataloger/golang/symbols.go

Lines changed: 122 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package golang
22

33
import (
4+
"bytes"
45
"debug/elf"
56
"debug/gosym"
67
"debug/macho"
8+
"encoding/binary"
79
"fmt"
810
"io"
911
"runtime/debug"
@@ -40,19 +42,113 @@ func getSymbols(r io.ReaderAt) (syms []binarySymbol, err error) {
4042
return nil, fmt.Errorf("unable to parse pclntab: %w", err)
4143
}
4244

45+
seen := make(map[string]struct{})
4346
for _, fn := range table.Funcs {
4447
if fn.Sym == nil {
4548
continue
4649
}
50+
seen[fn.Name] = struct{}{}
4751
syms = append(syms, binarySymbol{
4852
packagePath: fn.PackageName(),
4953
name: fn.Name,
5054
})
5155
}
5256

57+
// debug/gosym only exposes top-level functions; functions that the compiler inlined into their
58+
// callers are absent from table.Funcs even though their names are recorded in the pclntab funcname
59+
// table (used to reconstruct inlined frames in tracebacks). Recover those names so that a
60+
// vulnerable-but-inlined function (e.g. a small stdlib wrapper) is still reported as present.
61+
for _, name := range funcNameTable(pclntab) {
62+
if _, ok := seen[name]; ok {
63+
continue
64+
}
65+
pkgPath := packagePathFromSymbolName(name)
66+
if pkgPath == "" {
67+
continue
68+
}
69+
seen[name] = struct{}{}
70+
syms = append(syms, binarySymbol{packagePath: pkgPath, name: name})
71+
}
72+
5373
return syms, nil
5474
}
5575

76+
// packagePathFromSymbolName derives the owning package import path from a fully qualified symbol name.
77+
// The package path is everything up to the first "." that follows the final "/" — e.g.
78+
// "path/filepath.IsLocal" -> "path/filepath" and "golang.org/x/net/html.(*Tokenizer).Next" ->
79+
// "golang.org/x/net/html". Returns "" when the name has no package-qualifying dot.
80+
func packagePathFromSymbolName(name string) string {
81+
slash := strings.LastIndex(name, "/")
82+
dot := strings.IndexByte(name[slash+1:], '.')
83+
if dot < 0 {
84+
return ""
85+
}
86+
return name[:slash+1+dot]
87+
}
88+
89+
// funcNameTable returns every function name recorded in the pclntab's funcname table, including the
90+
// names of inlined functions that debug/gosym does not expose. It parses the pclntab header for the
91+
// Go 1.16+ layouts; on any unrecognized layout or out-of-bounds offset it returns nil (fail-soft), so
92+
// callers fall back to the debug/gosym function set. See the runtime's pcHeader / moduledata layout.
93+
func funcNameTable(pclntab []byte) []string {
94+
if len(pclntab) < 8 {
95+
return nil
96+
}
97+
98+
magic := binary.LittleEndian.Uint32(pclntab[0:4])
99+
// the field before funcnameOffset is textStart, which exists in the 1.18+ headers but not 1.16/1.17
100+
var hasTextStart bool
101+
switch magic {
102+
case 0xfffffff1, 0xfffffff0: // go1.20+, go1.18/1.19
103+
hasTextStart = true
104+
case 0xfffffffa: // go1.16/1.17
105+
hasTextStart = false
106+
default:
107+
return nil
108+
}
109+
110+
ptrSize := int(pclntab[7])
111+
if ptrSize != 4 && ptrSize != 8 {
112+
return nil
113+
}
114+
115+
readWord := func(idx int) (uint64, bool) {
116+
off := 8 + idx*ptrSize
117+
if off+ptrSize > len(pclntab) {
118+
return 0, false
119+
}
120+
if ptrSize == 8 {
121+
return binary.LittleEndian.Uint64(pclntab[off : off+8]), true
122+
}
123+
return uint64(binary.LittleEndian.Uint32(pclntab[off : off+4])), true
124+
}
125+
126+
// header words after (nfunc, nfiles): [textStart,] funcnameOffset, cuOffset, ...
127+
funcnameIdx := 2
128+
if hasTextStart {
129+
funcnameIdx = 3
130+
}
131+
funcnameOffset, ok1 := readWord(funcnameIdx)
132+
cuOffset, ok2 := readWord(funcnameIdx + 1)
133+
if !ok1 || !ok2 {
134+
return nil
135+
}
136+
137+
start, end := int(funcnameOffset), int(cuOffset)
138+
if start < 0 || end > len(pclntab) || start >= end {
139+
return nil
140+
}
141+
142+
var names []string
143+
for _, raw := range bytes.Split(pclntab[start:end], []byte{0}) {
144+
if len(raw) == 0 {
145+
continue
146+
}
147+
names = append(names, string(raw))
148+
}
149+
return names
150+
}
151+
56152
// readPclntab locates the pclntab and the start address of the text segment within the binary.
57153
func readPclntab(r io.ReaderAt) (pclntab []byte, textStart uint64, err error) {
58154
ident := make([]byte, 16)
@@ -106,11 +202,13 @@ func readPclntab(r io.ReaderAt) (pclntab []byte, textStart uint64, err error) {
106202

107203
// moduleSymbols attributes each extracted symbol to the module that owns it (by longest module path prefix
108204
// of the symbol's package path) and returns a sorted, deduplicated list of symbol names per module path.
109-
// Symbols from the "main" package are attributed to the main module. Stdlib and runtime symbols are not
110-
// attributed to any module.
111-
func moduleSymbols(symbols []binarySymbol, main *debug.Module, deps []*debug.Module) map[string][]string {
205+
// Symbols from the "main" package are attributed to the main module. Standard-library symbols (which belong
206+
// to no module) are collected separately and returned as the second value so they can be attached to the
207+
// synthetic "stdlib" package. Compiler/runtime-internal symbols that are neither module-owned nor a
208+
// recognizable stdlib import path are dropped.
209+
func moduleSymbols(symbols []binarySymbol, main *debug.Module, deps []*debug.Module) (byModule map[string][]string, stdlib []string) {
112210
if len(symbols) == 0 {
113-
return nil
211+
return nil, nil
114212
}
115213

116214
var modulePaths []string
@@ -138,6 +236,9 @@ func moduleSymbols(symbols []binarySymbol, main *debug.Module, deps []*debug.Mod
138236
}
139237
}
140238
if best == "" {
239+
if pkgPath != "main" && isStandardImportPath(pkgPath) {
240+
stdlib = append(stdlib, sym.name)
241+
}
141242
continue
142243
}
143244
results[best] = append(results[best], sym.name)
@@ -147,6 +248,22 @@ func moduleSymbols(symbols []binarySymbol, main *debug.Module, deps []*debug.Mod
147248
slices.Sort(names)
148249
results[modPath] = slices.Compact(names)
149250
}
251+
if len(stdlib) > 0 {
252+
slices.Sort(stdlib)
253+
stdlib = slices.Compact(stdlib)
254+
}
150255

151-
return results
256+
return results, stdlib
257+
}
258+
259+
// isStandardImportPath reports whether path is a Go standard-library import path. This mirrors the rule
260+
// the Go toolchain uses: a path is standard if the element before its first slash contains no dot (e.g.
261+
// "net/http", "runtime", "internal/abi"), which distinguishes it from module paths like
262+
// "github.com/foo/bar" whose leading element is a domain name.
263+
func isStandardImportPath(path string) bool {
264+
first := path
265+
if i := strings.Index(path, "/"); i >= 0 {
266+
first = path[:i]
267+
}
268+
return first != "" && !strings.Contains(first, ".")
152269
}

syft/pkg/cataloger/golang/symbols_test.go

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@ func Test_moduleSymbols(t *testing.T) {
1919
}
2020

2121
tests := []struct {
22-
name string
23-
symbols []binarySymbol
24-
expected map[string][]string
22+
name string
23+
symbols []binarySymbol
24+
expected map[string][]string
25+
expectedStdlib []string
2526
}{
2627
{
2728
name: "no symbols",
@@ -59,12 +60,18 @@ func Test_moduleSymbols(t *testing.T) {
5960
},
6061
},
6162
{
62-
name: "stdlib and runtime symbols are not attributed",
63+
name: "stdlib and runtime symbols are collected separately",
6364
symbols: []binarySymbol{
6465
{packagePath: "runtime", name: "runtime.main"},
6566
{packagePath: "net/http", name: "net/http.(*Client).Do"},
67+
{packagePath: "internal/abi", name: "internal/abi.(*Type).Kind"},
6668
},
6769
expected: map[string][]string{},
70+
expectedStdlib: []string{
71+
"internal/abi.(*Type).Kind",
72+
"net/http.(*Client).Do",
73+
"runtime.main",
74+
},
6875
},
6976
{
7077
name: "duplicate symbols are deduplicated",
@@ -82,7 +89,9 @@ func Test_moduleSymbols(t *testing.T) {
8289

8390
for _, tt := range tests {
8491
t.Run(tt.name, func(t *testing.T) {
85-
assert.Equal(t, tt.expected, moduleSymbols(tt.symbols, mainModule, deps))
92+
gotByModule, gotStdlib := moduleSymbols(tt.symbols, mainModule, deps)
93+
assert.Equal(t, tt.expected, gotByModule)
94+
assert.Equal(t, tt.expectedStdlib, gotStdlib)
8695
})
8796
}
8897
}

0 commit comments

Comments
 (0)