Skip to content

Commit e3b1de9

Browse files
jbajba
committed
internal/godoc: use fast codec
Use the fast encoder to write source to the database, under an experiment. DecodePackage doesn't need to check the experiment: it looks at the encoding type and invokes the right decoder for that type. Even after we remove the gob encoder, we have to leave the gob decoder around until everything is reprocessed. Change-Id: Icc443c60792305eeb8ae67dbd0168c92ee35de51 Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/264858 Trust: Jonathan Amsterdam <[email protected]> Run-TryBot: Jonathan Amsterdam <[email protected]> Reviewed-by: Julie Qiu <[email protected]>
1 parent 64a35f8 commit e3b1de9

File tree

5 files changed

+60
-25
lines changed

5 files changed

+60
-25
lines changed

‎internal/experiment.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ package internal
88
const (
99
ExperimentAltRequeue = "alt-requeue"
1010
ExperimentAutocomplete = "autocomplete"
11+
ExperimentFasterDecoding = "faster-decoding"
1112
ExperimentFrontendRenderDoc = "frontend-render-doc"
1213
ExperimentGetUnitWithOneQuery = "get-unit-with-one-query"
1314
ExperimentGoldmark = "goldmark"
@@ -22,6 +23,7 @@ const (
2223
var Experiments = map[string]string{
2324
ExperimentAltRequeue: "Requeue modules for reprocessing in a different order.",
2425
ExperimentAutocomplete: "Enable autocomplete with search.",
26+
ExperimentFasterDecoding: "Decode ASTs faster.",
2527
ExperimentFrontendRenderDoc: "Render documentation on the frontend if possible.",
2628
ExperimentGetUnitWithOneQuery: "Fetch data for GetUnit using a single query.",
2729
ExperimentGoldmark: "Enable the usage of rendering markdown using goldmark instead of blackfriday.",

‎internal/fetch/load.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ func loadPackageWithBuildContext(ctx context.Context, goos, goarch string, zipGo
125125
// it enough to make Render work.
126126
var src []byte
127127
if experiment.IsActive(ctx, internal.ExperimentInsertPackageSource) {
128-
src, err = docPkg.Encode()
128+
src, err = docPkg.Encode(ctx)
129129
if err != nil {
130130
return nil, err
131131
}

‎internal/godoc/encode.go

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package godoc
66

77
import (
88
"bytes"
9+
"context"
910
"encoding/gob"
1011
"errors"
1112
"fmt"
@@ -14,7 +15,9 @@ import (
1415
"io"
1516
"sort"
1617

18+
"golang.org/x/pkgsite/internal"
1719
"golang.org/x/pkgsite/internal/derrors"
20+
"golang.org/x/pkgsite/internal/experiment"
1821
"golang.org/x/pkgsite/internal/godoc/codec"
1922
)
2023

@@ -28,7 +31,7 @@ const (
2831

2932
// ErrInvalidEncodingType is returned when the data to DecodePackage has an
3033
// invalid encoding type.
31-
var ErrInvalidEncodingType = fmt.Errorf("want initial bytes to be %q but they aren't", gobEncodingType)
34+
var ErrInvalidEncodingType = fmt.Errorf("want initial bytes to be %q or %q but they aren't", gobEncodingType, fastEncodingType)
3235

3336
// Register ast types for gob, so it can decode concrete types that are stored
3437
// in interface variables.
@@ -97,9 +100,17 @@ func init() {
97100
// During its operation, Encode modifies the AST,
98101
// but it restores it to a state suitable for
99102
// rendering before it returns.
100-
func (p *Package) Encode() (_ []byte, err error) {
103+
func (p *Package) Encode(ctx context.Context) (_ []byte, err error) {
101104
defer derrors.Wrap(&err, "godoc.Package.Encode()")
102105

106+
if experiment.IsActive(ctx, internal.ExperimentFasterDecoding) {
107+
return p.fastEncode()
108+
} else {
109+
return p.gobEncode()
110+
}
111+
}
112+
113+
func (p *Package) gobEncode() (_ []byte, err error) {
103114
if p.renderCalled {
104115
return nil, errors.New("can't Encode after Render")
105116
}
@@ -128,10 +139,21 @@ func (p *Package) Encode() (_ []byte, err error) {
128139
func DecodePackage(data []byte) (_ *Package, err error) {
129140
defer derrors.Wrap(&err, "DecodePackage()")
130141

131-
if len(data) < encodingTypeLen || string(data[:encodingTypeLen]) != gobEncodingType {
142+
if len(data) < encodingTypeLen {
143+
return nil, ErrInvalidEncodingType
144+
}
145+
switch string(data[:encodingTypeLen]) {
146+
case gobEncodingType:
147+
return gobDecodePackage(data[encodingTypeLen:])
148+
case fastEncodingType:
149+
return fastDecodePackage(data[encodingTypeLen:])
150+
default:
132151
return nil, ErrInvalidEncodingType
133152
}
134-
dec := gob.NewDecoder(bytes.NewReader(data[encodingTypeLen:]))
153+
}
154+
155+
func gobDecodePackage(data []byte) (_ *Package, err error) {
156+
dec := gob.NewDecoder(bytes.NewReader(data))
135157
p := &Package{Fset: token.NewFileSet()}
136158
if err := p.Fset.Read(dec.Decode); err != nil {
137159
return nil, err
@@ -331,7 +353,7 @@ func isRelevantDecl(n interface{}) bool {
331353
}
332354
}
333355

334-
func (p *Package) FastEncode() (_ []byte, err error) {
356+
func (p *Package) fastEncode() (_ []byte, err error) {
335357
defer derrors.Wrap(&err, "godoc.Package.FastEncode()")
336358

337359
var buf bytes.Buffer
@@ -351,13 +373,10 @@ func (p *Package) FastEncode() (_ []byte, err error) {
351373
return buf.Bytes(), nil
352374
}
353375

354-
func FastDecodePackage(data []byte) (_ *Package, err error) {
376+
func fastDecodePackage(data []byte) (_ *Package, err error) {
355377
defer derrors.Wrap(&err, "FastDecodePackage()")
356378

357-
if len(data) < encodingTypeLen || string(data[:encodingTypeLen]) != fastEncodingType {
358-
return nil, fmt.Errorf("want initial bytes to be %q but they aren't", fastEncodingType)
359-
}
360-
dec := codec.NewDecoder(data[encodingTypeLen:])
379+
dec := codec.NewDecoder(data)
361380
x, err := dec.Decode()
362381
if err != nil {
363382
return nil, err

‎internal/godoc/encode_test.go

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package godoc
66

77
import (
88
"bytes"
9+
"context"
910
"fmt"
1011
"go/ast"
1112
"go/parser"
@@ -19,26 +20,28 @@ import (
1920
"testing"
2021

2122
"github.com/google/go-cmp/cmp"
23+
"golang.org/x/pkgsite/internal"
24+
"golang.org/x/pkgsite/internal/experiment"
2225
)
2326

2427
var packageToTest string = filepath.Join(runtime.GOROOT(), "src", "net", "http")
2528

2629
func TestEncodeDecodePackage(t *testing.T) {
2730
// Verify that we can encode and decode the Go files in this directory.
28-
p, err := packageForDir(packageToTest, true)
31+
p, err := packageForDir(".", true)
2932
if err != nil {
3033
t.Fatal(err)
3134
}
3235

33-
data, err := p.Encode()
36+
data, err := p.Encode(context.Background())
3437
if err != nil {
3538
t.Fatal(err)
3639
}
3740
p2, err := DecodePackage(data)
3841
if err != nil {
3942
t.Fatal(err)
4043
}
41-
data2, err := p2.Encode()
44+
data2, err := p2.Encode(context.Background())
4245
if err != nil {
4346
t.Fatal(err)
4447
}
@@ -49,6 +52,7 @@ func TestEncodeDecodePackage(t *testing.T) {
4952

5053
func TestObjectIdentity(t *testing.T) {
5154
// Check that encoding and decoding preserves object identity.
55+
ctx := context.Background()
5256
const file = `
5357
package p
5458
var a int
@@ -75,7 +79,7 @@ func main() { a = 1 }
7579

7680
p := NewPackage(fset, "linux", "amd64", nil)
7781
p.AddFile(f, false)
78-
data, err := p.Encode()
82+
data, err := p.Encode(ctx)
7983
if err != nil {
8084
t.Fatal(err)
8185
}
@@ -114,7 +118,7 @@ func BenchmarkRemovingAST(b *testing.B) {
114118
if err != nil {
115119
b.Fatal(err)
116120
}
117-
data, err := p.Encode()
121+
data, err := p.Encode(context.Background())
118122
if err != nil {
119123
b.Fatal(err)
120124
}
@@ -136,11 +140,11 @@ func TestFastEncode(t *testing.T) {
136140
}
137141
var want, got bytes.Buffer
138142
printPackage(&want, p)
139-
data, err := p.FastEncode()
143+
data, err := p.Encode(experiment.NewContext(context.Background(), internal.ExperimentFasterDecoding))
140144
if err != nil {
141145
t.Fatal(err)
142146
}
143-
p2, err := FastDecodePackage(data)
147+
p2, err := DecodePackage(data)
144148
if err != nil {
145149
t.Fatal(err)
146150
}
@@ -283,15 +287,15 @@ func BenchmarkEncoding(b *testing.B) {
283287
}
284288
b.Run("gob", func(b *testing.B) {
285289
for i := 0; i < b.N; i++ {
286-
_, err := p.Encode()
290+
_, err := p.gobEncode()
287291
if err != nil {
288292
b.Fatal(err)
289293
}
290294
}
291295
})
292296
b.Run("fast", func(b *testing.B) {
293297
for i := 0; i < b.N; i++ {
294-
_, err := p.FastEncode()
298+
_, err := p.fastEncode()
295299
if err != nil {
296300
b.Fatal(err)
297301
}
@@ -306,26 +310,26 @@ func BenchmarkDecoding(b *testing.B) {
306310
b.Fatal(err)
307311
}
308312
b.Run("gob", func(b *testing.B) {
309-
data, err := p.Encode()
313+
data, err := p.gobEncode()
310314
if err != nil {
311315
b.Fatal(err)
312316
}
313317
b.ResetTimer()
314318
for i := 0; i < b.N; i++ {
315-
_, err := DecodePackage(data)
319+
_, err := gobDecodePackage(data[encodingTypeLen:])
316320
if err != nil {
317321
b.Fatal(err)
318322
}
319323
}
320324
})
321325
b.Run("fast", func(b *testing.B) {
322-
data, err := p.FastEncode()
326+
data, err := p.fastEncode()
323327
if err != nil {
324328
b.Fatal(err)
325329
}
326330
b.ResetTimer()
327331
for i := 0; i < b.N; i++ {
328-
_, err := FastDecodePackage(data)
332+
_, err := fastDecodePackage(data[encodingTypeLen:])
329333
if err != nil {
330334
b.Fatal(err)
331335
}

‎internal/godoc/render_test.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,23 @@ import (
1111
"testing"
1212

1313
"github.com/google/go-cmp/cmp"
14+
"golang.org/x/pkgsite/internal"
15+
"golang.org/x/pkgsite/internal/experiment"
1416
"golang.org/x/pkgsite/internal/source"
1517
"golang.org/x/pkgsite/internal/testing/sample"
1618
)
1719

1820
func TestRender(t *testing.T) {
1921
ctx := context.Background()
22+
t.Run("gob", func(t *testing.T) {
23+
testRender(t, ctx)
24+
})
25+
t.Run("fast", func(t *testing.T) {
26+
testRender(t, experiment.NewContext(ctx, internal.ExperimentFasterDecoding))
27+
})
28+
}
2029

30+
func testRender(t *testing.T, ctx context.Context) {
2131
si := source.NewGitHubInfo(sample.ModulePath, "", "abcde")
2232
mi := &ModuleInfo{
2333
ModulePath: sample.ModulePath,
@@ -71,7 +81,7 @@ func TestRender(t *testing.T) {
7181
if err != nil {
7282
t.Fatal(err)
7383
}
74-
bytes, err := p.Encode()
84+
bytes, err := p.Encode(ctx)
7585
if err != nil {
7686
t.Fatal(err)
7787
}

0 commit comments

Comments
 (0)