bench_test.go 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146
  1. // Copyright (c) 2020, The Garble Authors.
  2. // See LICENSE for licensing information.
  3. package main
  4. import (
  5. _ "embed"
  6. "flag"
  7. "fmt"
  8. "os"
  9. "os/exec"
  10. "path/filepath"
  11. "regexp"
  12. "runtime"
  13. "strconv"
  14. "strings"
  15. "testing"
  16. "time"
  17. qt "github.com/frankban/quicktest"
  18. )
  19. //go:embed testdata/bench/main.go
  20. var benchSourceMain []byte
  21. var (
  22. rxBuiltRuntime = regexp.MustCompile(`(?m)^runtime$`)
  23. rxBuiltMain = regexp.MustCompile(`(?m)^test/main$`)
  24. )
  25. // BenchmarkBuild is a benchmark for 'garble build' on a fairly simple
  26. // main package with a handful of standard library depedencies.
  27. //
  28. // We use a real garble binary and exec it, to simulate what the real user would
  29. // run. The real obfuscation and compilation will happen in sub-processes
  30. // anyway, so skipping one exec layer doesn't help us in any way.
  31. //
  32. // The benchmark isn't parallel, because in practice users build once at a time,
  33. // and each build already spawns concurrent processes and goroutines to do work.
  34. //
  35. // At the moment, each iteration takes 1-2s on a laptop, so we can't make the
  36. // benchmark include any more features unless we make it significantly faster.
  37. func BenchmarkBuild(b *testing.B) {
  38. // As of Go 1.17, using -benchtime=Nx with N larger than 1 results in two
  39. // calls to BenchmarkBuild, with the first having b.N==1 to discover
  40. // sub-benchmarks. Unfortunately, we do a significant amount of work both
  41. // during setup and during that first iteration, which is pointless.
  42. // To avoid that, detect the scenario in a hacky way, and return early.
  43. // See https://github.com/golang/go/issues/32051.
  44. benchtime := flag.Lookup("test.benchtime").Value.String()
  45. if b.N == 1 && strings.HasSuffix(benchtime, "x") && benchtime != "1x" {
  46. return
  47. }
  48. garbleBin := filepath.Join(b.TempDir(), "garble")
  49. if runtime.GOOS == "windows" {
  50. garbleBin += ".exe"
  51. }
  52. err := exec.Command("go", "build", "-o="+garbleBin).Run()
  53. qt.Assert(b, err, qt.IsNil)
  54. // We collect extra metrics.
  55. var memoryAllocs, cachedTime, systemTime int64
  56. outputBin := filepath.Join(b.TempDir(), "output")
  57. sourceDir := filepath.Join(b.TempDir(), "src")
  58. err = os.Mkdir(sourceDir, 0o777)
  59. qt.Assert(b, err, qt.IsNil)
  60. writeSourceFile := func(name string, content []byte) {
  61. err := os.WriteFile(filepath.Join(sourceDir, name), content, 0o666)
  62. qt.Assert(b, err, qt.IsNil)
  63. }
  64. writeSourceFile("go.mod", []byte("module test/main"))
  65. writeSourceFile("main.go", benchSourceMain)
  66. rxGarbleAllocs := regexp.MustCompile(`(?m)^garble allocs: ([0-9]+)`)
  67. b.ResetTimer()
  68. b.StopTimer()
  69. for i := 0; i < b.N; i++ {
  70. // First we do a fresh build, using a new GOCACHE.
  71. // and the second does an incremental rebuild reusing the cache.
  72. gocache, err := os.MkdirTemp(b.TempDir(), "gocache-*")
  73. qt.Assert(b, err, qt.IsNil)
  74. env := append(os.Environ(),
  75. "GOGARBLE=*",
  76. "GOCACHE="+gocache,
  77. "GARBLE_WRITE_ALLOCS=true",
  78. )
  79. args := []string{"build", "-v", "-o=" + outputBin, sourceDir}
  80. for _, cached := range []bool{false, true} {
  81. // The cached rebuild will reuse all dependencies,
  82. // but rebuild the main package itself.
  83. if cached {
  84. writeSourceFile("rebuild.go", []byte(fmt.Sprintf("package main\nvar v%d int", i)))
  85. }
  86. cmd := exec.Command(garbleBin, args...)
  87. cmd.Env = env
  88. cmd.Dir = sourceDir
  89. cachedStart := time.Now()
  90. b.StartTimer()
  91. out, err := cmd.CombinedOutput()
  92. b.StopTimer()
  93. if cached {
  94. cachedTime += time.Since(cachedStart).Nanoseconds()
  95. }
  96. qt.Assert(b, err, qt.IsNil, qt.Commentf("output: %s", out))
  97. if !cached {
  98. // Ensure that we built all packages, as expected.
  99. qt.Assert(b, rxBuiltRuntime.Match(out), qt.IsTrue)
  100. } else {
  101. // Ensure that we only rebuilt the main package, as expected.
  102. qt.Assert(b, rxBuiltRuntime.Match(out), qt.IsFalse)
  103. }
  104. qt.Assert(b, rxBuiltMain.Match(out), qt.IsTrue)
  105. matches := rxGarbleAllocs.FindAllSubmatch(out, -1)
  106. if !cached {
  107. // The non-cached version should have at least a handful of
  108. // sub-processes; catch if our logic breaks.
  109. qt.Assert(b, len(matches) > 5, qt.IsTrue)
  110. }
  111. for _, match := range matches {
  112. allocs, err := strconv.ParseInt(string(match[1]), 10, 64)
  113. qt.Assert(b, err, qt.IsNil)
  114. memoryAllocs += allocs
  115. }
  116. systemTime += int64(cmd.ProcessState.SystemTime())
  117. }
  118. }
  119. // We can't use "allocs/op" as it's reserved for ReportAllocs.
  120. b.ReportMetric(float64(memoryAllocs)/float64(b.N), "mallocs/op")
  121. b.ReportMetric(float64(cachedTime)/float64(b.N), "cached-ns/op")
  122. b.ReportMetric(float64(systemTime)/float64(b.N), "sys-ns/op")
  123. info, err := os.Stat(outputBin)
  124. if err != nil {
  125. b.Fatal(err)
  126. }
  127. b.ReportMetric(float64(info.Size()), "bin-B")
  128. }