shared.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389
  1. // Copyright (c) 2020, The Garble Authors.
  2. // See LICENSE for licensing information.
  3. package main
  4. import (
  5. "bytes"
  6. "encoding/gob"
  7. "encoding/json"
  8. "fmt"
  9. "log"
  10. "os"
  11. "os/exec"
  12. "path/filepath"
  13. "strings"
  14. "time"
  15. "golang.org/x/mod/module"
  16. )
  17. // sharedCache is shared as a read-only cache between the many garble toolexec
  18. // sub-processes.
  19. //
  20. // Note that we fill this cache once from the root process in saveListedPackages,
  21. // store it into a temporary file via gob encoding, and then reuse that file
  22. // in each of the garble toolexec sub-processes.
  23. type sharedCache struct {
  24. ExecPath string // absolute path to the garble binary being used
  25. ForwardBuildFlags []string // build flags fed to the original "garble ..." command
  26. // ListedPackages contains data obtained via 'go list -json -export -deps'.
  27. // This allows us to obtain the non-obfuscated export data of all dependencies,
  28. // useful for type checking of the packages as we obfuscate them.
  29. ListedPackages map[string]*listedPackage
  30. // We can't rely on the module version to exist,
  31. // because it's missing in local builds without 'go install'.
  32. // For now, use 'go tool buildid' on the garble binary.
  33. // Just like Go's own cache, we use hex-encoded sha256 sums.
  34. // Once https://github.com/golang/go/issues/37475 is fixed,
  35. // we can likely just use that.
  36. BinaryContentID []byte
  37. GOGARBLE string
  38. // Filled directly from "go env".
  39. // Remember to update the exec call when adding or removing names.
  40. GoEnv struct {
  41. GOOS string // i.e. the GOOS build target
  42. GOPRIVATE string
  43. GOMOD string
  44. GOVERSION string
  45. GOCACHE string
  46. }
  47. }
  48. var cache *sharedCache
  49. // loadSharedCache the shared data passed from the entry garble process
  50. func loadSharedCache() error {
  51. if cache != nil {
  52. panic("shared cache loaded twice?")
  53. }
  54. startTime := time.Now()
  55. f, err := os.Open(filepath.Join(sharedTempDir, "main-cache.gob"))
  56. if err != nil {
  57. return fmt.Errorf(`cannot open shared file: %v; did you run "garble [command]"?`, err)
  58. }
  59. defer func() {
  60. log.Printf("shared cache loaded in %s from %s", debugSince(startTime), f.Name())
  61. }()
  62. defer f.Close()
  63. if err := gob.NewDecoder(f).Decode(&cache); err != nil {
  64. return fmt.Errorf("cannot decode shared file: %v", err)
  65. }
  66. return nil
  67. }
  68. // saveSharedCache creates a temporary directory to share between garble processes.
  69. // This directory also includes the gob-encoded cache global.
  70. func saveSharedCache() (string, error) {
  71. if cache == nil {
  72. panic("saving a missing cache?")
  73. }
  74. dir, err := os.MkdirTemp("", "garble-shared")
  75. if err != nil {
  76. return "", err
  77. }
  78. sharedCache := filepath.Join(dir, "main-cache.gob")
  79. if err := writeGobExclusive(sharedCache, &cache); err != nil {
  80. return "", err
  81. }
  82. return dir, nil
  83. }
  84. func createExclusive(name string) (*os.File, error) {
  85. return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o666)
  86. }
  87. // TODO(mvdan): consider using proper atomic file writes.
  88. // Or possibly even "lockedfile", mimicking cmd/go.
  89. func writeFileExclusive(name string, data []byte) error {
  90. f, err := createExclusive(name)
  91. if err != nil {
  92. return err
  93. }
  94. _, err = f.Write(data)
  95. if err2 := f.Close(); err == nil {
  96. err = err2
  97. }
  98. return err
  99. }
  100. func writeGobExclusive(name string, val any) error {
  101. f, err := createExclusive(name)
  102. if err != nil {
  103. return err
  104. }
  105. if err := gob.NewEncoder(f).Encode(val); err != nil {
  106. return err
  107. }
  108. if err2 := f.Close(); err == nil {
  109. err = err2
  110. }
  111. return err
  112. }
  113. // listedPackage contains the 'go list -json -export' fields obtained by the
  114. // root process, shared with all garble sub-processes via a file.
  115. type listedPackage struct {
  116. Name string
  117. ImportPath string
  118. ForTest string
  119. Export string
  120. BuildID string
  121. Deps []string
  122. ImportMap map[string]string
  123. Standard bool
  124. Dir string
  125. GoFiles []string
  126. Imports []string
  127. Incomplete bool
  128. // The fields below are not part of 'go list', but are still reused
  129. // between garble processes. Use "Garble" as a prefix to ensure no
  130. // collisions with the JSON fields from 'go list'.
  131. // GarbleActionID is a hash combining the Action ID from BuildID,
  132. // with Garble's own inputs as per addGarbleToHash.
  133. // It is set even when ToObfuscate is false, as it is also used for random
  134. // seeds and build cache paths, and not just to obfuscate names.
  135. GarbleActionID []byte `json:"-"`
  136. // ToObfuscate records whether the package should be obfuscated.
  137. ToObfuscate bool `json:"-"`
  138. }
  139. func (p *listedPackage) obfuscatedImportPath() string {
  140. if p.Name == "main" {
  141. panic("main packages should never need to obfuscate their import paths")
  142. }
  143. // We can't obfuscate the embed package's import path,
  144. // as the toolchain expects to recognize the package by it.
  145. if p.ImportPath == "embed" || !p.ToObfuscate {
  146. return p.ImportPath
  147. }
  148. newPath := hashWithPackage(p, p.ImportPath)
  149. log.Printf("import path %q hashed with %x to %q", p.ImportPath, p.GarbleActionID, newPath)
  150. return newPath
  151. }
  152. // appendListedPackages gets information about the current package
  153. // and all of its dependencies
  154. func appendListedPackages(packages []string, withDeps bool) error {
  155. startTime := time.Now()
  156. // TODO: perhaps include all top-level build flags set by garble,
  157. // including -buildvcs=false.
  158. // They shouldn't affect "go list" here, but might as well be consistent.
  159. args := []string{"list", "-json", "-export", "-trimpath", "-e"}
  160. if withDeps {
  161. args = append(args, "-deps")
  162. }
  163. args = append(args, cache.ForwardBuildFlags...)
  164. args = append(args, packages...)
  165. cmd := exec.Command("go", args...)
  166. defer func() {
  167. log.Printf("original build info obtained in %s via: go %s", debugSince(startTime), strings.Join(args, " "))
  168. }()
  169. stdout, err := cmd.StdoutPipe()
  170. if err != nil {
  171. return err
  172. }
  173. var stderr bytes.Buffer
  174. cmd.Stderr = &stderr
  175. if err := cmd.Start(); err != nil {
  176. return fmt.Errorf("go list error: %v", err)
  177. }
  178. dec := json.NewDecoder(stdout)
  179. if cache.ListedPackages == nil {
  180. cache.ListedPackages = make(map[string]*listedPackage)
  181. }
  182. for dec.More() {
  183. var pkg listedPackage
  184. if err := dec.Decode(&pkg); err != nil {
  185. return err
  186. }
  187. // Note that we use the `-e` flag above with `go list`.
  188. // If a package fails to load, the Incomplete and Error fields will be set.
  189. // We still record failed packages in the ListedPackages map,
  190. // because some like crypto/internal/boring/fipstls simply fall under
  191. // "build constraints exclude all Go files" and can be ignored.
  192. // Real build errors will still be surfaced by `go build -toolexec` later.
  193. if cache.ListedPackages[pkg.ImportPath] != nil {
  194. return fmt.Errorf("duplicate package: %q", pkg.ImportPath)
  195. }
  196. if pkg.BuildID != "" {
  197. actionID := decodeHash(splitActionID(pkg.BuildID))
  198. pkg.GarbleActionID = addGarbleToHash(actionID)
  199. }
  200. cache.ListedPackages[pkg.ImportPath] = &pkg
  201. }
  202. if err := cmd.Wait(); err != nil {
  203. return fmt.Errorf("go list error: %v: %s", err, stderr.Bytes())
  204. }
  205. anyToObfuscate := false
  206. for path, pkg := range cache.ListedPackages {
  207. // If "GOGARBLE=foo/bar", "foo/bar_test" should also match.
  208. if pkg.ForTest != "" {
  209. path = pkg.ForTest
  210. }
  211. // Test main packages like "foo/bar.test" are always obfuscated,
  212. // just like main packages.
  213. switch {
  214. case cannotObfuscate[path]:
  215. // We don't support obfuscating these yet.
  216. case pkg.Incomplete:
  217. // We can't obfuscate packages which weren't loaded.
  218. case pkg.Name == "main" && strings.HasSuffix(path, ".test"),
  219. path == "command-line-arguments",
  220. strings.HasPrefix(path, "plugin/unnamed"),
  221. module.MatchPrefixPatterns(cache.GOGARBLE, path):
  222. pkg.ToObfuscate = true
  223. anyToObfuscate = true
  224. }
  225. }
  226. // Don't error if the user ran: GOGARBLE='*' garble build runtime
  227. if !anyToObfuscate && !module.MatchPrefixPatterns(cache.GOGARBLE, "runtime") {
  228. return fmt.Errorf("GOGARBLE=%q does not match any packages to be built", cache.GOGARBLE)
  229. }
  230. return nil
  231. }
  232. // cannotObfuscate is a list of some standard library packages we currently
  233. // cannot obfuscate.
  234. //
  235. // TODO: investigate and resolve each one of these
  236. var cannotObfuscate = map[string]bool{
  237. // "undefined reference" errors at link time
  238. "time": true,
  239. // "//go:linkname must refer to declared function or variable"
  240. "syscall": true,
  241. // "unknown pc" crashes on windows in the cgo test otherwise
  242. "runtime/cgo": true,
  243. // We do not support obfuscating the runtime nor its dependencies.
  244. // Obtained from "go list -deps runtime" as of June 29th.
  245. // Note that the same command on Go 1.18 results in the same list.
  246. "internal/goarch": true,
  247. "unsafe": true,
  248. "internal/abi": true,
  249. "internal/cpu": true,
  250. "internal/bytealg": true,
  251. "internal/goexperiment": true,
  252. "internal/goos": true,
  253. "runtime/internal/atomic": true,
  254. "runtime/internal/math": true,
  255. "runtime/internal/sys": true,
  256. "runtime/internal/syscall": true,
  257. "runtime": true,
  258. }
  259. var listedRuntimeLinknamed = false
  260. // listPackage gets the listedPackage information for a certain package
  261. func listPackage(path string) (*listedPackage, error) {
  262. if path == curPkg.ImportPath {
  263. return curPkg, nil
  264. }
  265. // If the path is listed in the top-level ImportMap, use its mapping instead.
  266. // This is a common scenario when dealing with vendored packages in GOROOT.
  267. // The map is flat, so we don't need to recurse.
  268. if path2 := curPkg.ImportMap[path]; path2 != "" {
  269. path = path2
  270. }
  271. pkg, ok := cache.ListedPackages[path]
  272. // The runtime may list any package in std, even those it doesn't depend on.
  273. // This is due to how it linkname-implements std packages,
  274. // such as sync/atomic or reflect, without importing them in any way.
  275. // If ListedPackages lacks such a package we fill it with "std".
  276. if curPkg.ImportPath == "runtime" {
  277. if ok {
  278. return pkg, nil
  279. }
  280. if listedRuntimeLinknamed {
  281. panic(fmt.Sprintf("package %q still missing after go list call", path))
  282. }
  283. startTime := time.Now()
  284. // Obtained via scripts/runtime-linknamed-nodeps.sh as of June 29th.
  285. runtimeLinknamed := []string{
  286. "crypto/internal/boring",
  287. "crypto/internal/boring/bcache",
  288. "crypto/internal/boring/fipstls",
  289. "crypto/x509/internal/macos",
  290. "internal/poll",
  291. "internal/reflectlite",
  292. "net",
  293. "os",
  294. "os/signal",
  295. "plugin",
  296. "reflect",
  297. "runtime/debug",
  298. "runtime/metrics",
  299. "runtime/pprof",
  300. "runtime/trace",
  301. "sync",
  302. "sync/atomic",
  303. "syscall",
  304. "syscall/js",
  305. "time",
  306. }
  307. missing := make([]string, 0, len(runtimeLinknamed))
  308. for _, linknamed := range runtimeLinknamed {
  309. switch {
  310. case cache.ListedPackages[linknamed] != nil:
  311. // We already have it; skip.
  312. case cache.GoEnv.GOOS != "js" && linknamed == "syscall/js":
  313. // GOOS-specific package.
  314. case cache.GoEnv.GOOS != "darwin" && linknamed == "crypto/x509/internal/macos":
  315. // GOOS-specific package.
  316. default:
  317. missing = append(missing, linknamed)
  318. }
  319. }
  320. // We don't need any information about their dependencies, in this case.
  321. if err := appendListedPackages(missing, false); err != nil {
  322. panic(err) // should never happen
  323. }
  324. pkg, ok := cache.ListedPackages[path]
  325. if !ok {
  326. panic(fmt.Sprintf("runtime listed a std package we can't find: %s", path))
  327. }
  328. listedRuntimeLinknamed = true
  329. log.Printf("listed %d missing runtime-linknamed packages in %s", len(missing), debugSince(startTime))
  330. return pkg, nil
  331. }
  332. // Packages other than runtime can list any package,
  333. // as long as they depend on it directly or indirectly.
  334. if !ok {
  335. return nil, fmt.Errorf("path not found in listed packages: %s", path)
  336. }
  337. for _, dep := range curPkg.Deps {
  338. if dep == pkg.ImportPath {
  339. return pkg, nil
  340. }
  341. }
  342. return nil, fmt.Errorf("refusing to list non-dependency package: %s", path)
  343. }