Skip to content

Commit

Permalink
protobuf: During generation, copy protobuf tags back
Browse files Browse the repository at this point in the history 
The protobuf tags contain the assigned tag id, which then ensures
subsequent generation is consistently tagging (tags don't change across
generations unless someone deletes the protobuf tag).

In addition, generate final proto IDL that is free of gogoproto
extensions for ease of generation into other languages.

Add a flag --keep-gogoproto which preserves the gogoproto extensions in
the final IDL.
  • Loading branch information
@smarterclayton
smarterclayton committed Jan 26, 2016
1 parent e0e305c commit 14a3aaf
Show file tree
Hide file tree
Showing 6 changed files with 403 additions and 53 deletions.
90 changes: 70 additions & 20 deletions cmd/libs/go2idl/go-to-protobuf/protobuf/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type Generator struct {
Conditional string
Clean bool
OnlyIDL bool
KeepGogoproto bool
SkipGeneratedRewrite bool
DropEmbeddedFields string
}
Expand Down Expand Up @@ -77,6 +78,7 @@ func (g *Generator) BindFlags(flag *flag.FlagSet) {
flag.StringVar(&g.Conditional, "conditional", g.Conditional, "An optional Golang build tag condition to add to the generated Go code")
flag.BoolVar(&g.Clean, "clean", g.Clean, "If true, remove all generated files for the specified Packages.")
flag.BoolVar(&g.OnlyIDL, "only-idl", g.OnlyIDL, "If true, only generate the IDL for each package.")
flag.BoolVar(&g.KeepGogoproto, "keep-gogoproto", g.KeepGogoproto, "If true, the generated IDL will contain gogoprotobuf extensions which are normally removed")
flag.BoolVar(&g.SkipGeneratedRewrite, "skip-generated-rewrite", g.SkipGeneratedRewrite, "If true, skip fixing up the generated.pb.go file (debugging only).")
flag.StringVar(&g.DropEmbeddedFields, "drop-embedded-fields", g.DropEmbeddedFields, "Comma-delimited list of embedded Go types to omit from generated protobufs")
}
Expand Down Expand Up @@ -206,8 +208,11 @@ func Run(g *Generator) {

for _, outputPackage := range outputPackages {
p := outputPackage.(*protobufPackage)

path := filepath.Join(g.OutputBase, p.ImportPath())
outputPath := filepath.Join(g.OutputBase, p.OutputPath())

// generate the gogoprotobuf protoc
cmd := exec.Command("protoc", append(args, path)...)
out, err := cmd.CombinedOutput()
if len(out) > 0 {
Expand All @@ -217,29 +222,74 @@ func Run(g *Generator) {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err)
}
if !g.SkipGeneratedRewrite {
if err := RewriteGeneratedGogoProtobufFile(outputPath, p.GoPackageName(), p.HasGoType, buf.Bytes()); err != nil {
log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
}

cmd := exec.Command("goimports", "-w", outputPath)
out, err := cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
}
if g.SkipGeneratedRewrite {
continue
}

// alter the generated protobuf file to remove the generated types (but leave the serializers) and rewrite the
// package statement to match the desired package name
if err := RewriteGeneratedGogoProtobufFile(outputPath, p.GoPackageName(), p.ExtractGeneratedType, buf.Bytes()); err != nil {
log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
}

// sort imports
cmd = exec.Command("goimports", "-w", outputPath)
out, err = cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
}

// format and simplify the generated file
cmd = exec.Command("gofmt", "-s", "-w", outputPath)
out, err = cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to apply gofmt for %s: %v", p.PackageName, err)
}
}

if g.SkipGeneratedRewrite {
return
}

if !g.KeepGogoproto {
// generate, but do so without gogoprotobuf extensions
for _, outputPackage := range outputPackages {
p := outputPackage.(*protobufPackage)
p.OmitGogo = true
}
if err := c.ExecutePackages(g.OutputBase, outputPackages); err != nil {
log.Fatalf("Failed executing generator: %v", err)
}
}

for _, outputPackage := range outputPackages {
p := outputPackage.(*protobufPackage)

if len(p.StructTags) == 0 {
continue
}

pattern := filepath.Join(g.OutputBase, p.PackagePath, "*.go")
files, err := filepath.Glob(pattern)
if err != nil {
log.Fatalf("Can't glob pattern %q: %v", pattern, err)
}

cmd = exec.Command("gofmt", "-s", "-w", outputPath)
out, err = cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
for _, s := range files {
if strings.HasSuffix(s, "_test.go") {
continue
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
if err := RewriteTypesWithProtobufStructTags(s, p.StructTags); err != nil {
log.Fatalf("Unable to rewrite with struct tags %s: %v", s, err)
}
}
}
Expand Down
83 changes: 54 additions & 29 deletions cmd/libs/go2idl/go-to-protobuf/protobuf/generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,16 @@ type genProtoIDL struct {
imports *ImportTracker

generateAll bool
omitGogo bool
omitFieldTypes map[types.Name]struct{}
}

func (g *genProtoIDL) PackageVars(c *generator.Context) []string {
if g.omitGogo {
return []string{
fmt.Sprintf("option go_package = %q;", g.localGoPackage.Name),
}
}
return []string{
"option (gogoproto.marshaler_all) = true;",
"option (gogoproto.sizer_all) = true;",
Expand Down Expand Up @@ -117,7 +123,15 @@ func isProtoable(seen map[*types.Type]bool, t *types.Type) bool {
}

func (g *genProtoIDL) Imports(c *generator.Context) (imports []string) {
return g.imports.ImportLines()
lines := []string{}
// TODO: this could be expressed more cleanly
for _, line := range g.imports.ImportLines() {
if g.omitGogo && line == "github.com/gogo/protobuf/gogoproto/gogo.proto" {
continue
}
lines = append(lines, line)
}
return lines
}

// GenerateType makes the body of a file implementing a set for type t.
Expand All @@ -130,7 +144,9 @@ func (g *genProtoIDL) GenerateType(c *generator.Context, t *types.Type, w io.Wri

localGoPackage: g.localGoPackage.Package,
},
localPackage: g.localPackage,
localPackage: g.localPackage,

omitGogo: g.omitGogo,
omitFieldTypes: g.omitFieldTypes,

t: t,
Expand Down Expand Up @@ -201,6 +217,7 @@ func (p protobufLocator) ProtoTypeFor(t *types.Type) (*types.Type, error) {
type bodyGen struct {
locator ProtobufLocator
localPackage types.Name
omitGogo bool
omitFieldTypes map[types.Name]struct{}

t *types.Type
Expand Down Expand Up @@ -228,14 +245,18 @@ func (b bodyGen) doStruct(sw *generator.SnippetWriter) error {
switch key {
case "marshal":
if v == "false" {
options = append(options,
"(gogoproto.marshaler) = false",
"(gogoproto.unmarshaler) = false",
"(gogoproto.sizer) = false",
)
if !b.omitGogo {
options = append(options,
"(gogoproto.marshaler) = false",
"(gogoproto.unmarshaler) = false",
"(gogoproto.sizer) = false",
)
}
}
default:
options = append(options, fmt.Sprintf("%s = %s", key, v))
if !b.omitGogo || !strings.HasPrefix(key, "(gogoproto.") {
options = append(options, fmt.Sprintf("%s = %s", key, v))
}
}
case k == "protobuf.embed":
fields = []protoField{
Expand Down Expand Up @@ -289,14 +310,19 @@ func (b bodyGen) doStruct(sw *generator.SnippetWriter) error {
}
sw.Do(`$.Type|local$ $.Name$ = $.Tag$`, field)
if len(field.Extras) > 0 {
fmt.Fprintf(out, " [")
extras := []string{}
for k, v := range field.Extras {
if b.omitGogo && strings.HasPrefix(k, "(gogoproto.") {
continue
}
extras = append(extras, fmt.Sprintf("%s = %s", k, v))
}
sort.Sort(sort.StringSlice(extras))
fmt.Fprint(out, strings.Join(extras, ", "))
fmt.Fprintf(out, "]")
if len(extras) > 0 {
fmt.Fprintf(out, " [")
fmt.Fprint(out, strings.Join(extras, ", "))
fmt.Fprintf(out, "]")
}
}
fmt.Fprintf(out, ";\n")
if i != len(fields)-1 {
Expand Down Expand Up @@ -459,24 +485,19 @@ func protobufTagToField(tag string, field *protoField, m types.Member, t *types.
Kind: typesKindProtobuf,
}
} else {
field.Type = &types.Type{
Name: types.Name{
Name: parts[0],
Package: localPackage.Package,
Path: localPackage.Path,
},
Kind: typesKindProtobuf,
switch parts[0] {
case "varint", "bytes", "fixed64":
default:
field.Type = &types.Type{
Name: types.Name{
Name: parts[0],
Package: localPackage.Package,
Path: localPackage.Path,
},
Kind: typesKindProtobuf,
}
}
}
switch parts[2] {
case "rep":
field.Repeated = true
case "opt":
field.Optional = true
case "req":
default:
return fmt.Errorf("member %q of %q malformed 'protobuf' tag, field mode is %q not recognized\n", m.Name, t.Name, parts[2])
}
field.OptionalSet = true

protoExtra := make(map[string]string)
Expand All @@ -485,7 +506,11 @@ func protobufTagToField(tag string, field *protoField, m types.Member, t *types.
if len(parts) != 2 {
return fmt.Errorf("member %q of %q malformed 'protobuf' tag, tag %d should be key=value, got %q\n", m.Name, t.Name, i+4, extra)
}
protoExtra[parts[0]] = parts[1]
switch parts[0] {
case "casttype":
parts[0] = fmt.Sprintf("(gogoproto.%s)", parts[0])
protoExtra[parts[0]] = parts[1]
}
}

field.Extras = protoExtra
Expand Down Expand Up @@ -526,7 +551,7 @@ func membersToFields(locator ProtobufLocator, t *types.Type, localPackage types.
if len(field.Name) == 0 && len(parts[0]) != 0 {
field.Name = parts[0]
}
if field.Name == "-" {
if field.Tag == -1 && field.Name == "-" {
continue
}
}
Expand Down
49 changes: 49 additions & 0 deletions cmd/libs/go2idl/go-to-protobuf/protobuf/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,13 @@ package protobuf

import (
"fmt"
"log"
"os"
"path/filepath"
"reflect"
"strings"

"k8s.io/kubernetes/third_party/golang/go/ast"

"k8s.io/kubernetes/cmd/libs/go2idl/generator"
"k8s.io/kubernetes/cmd/libs/go2idl/types"
Expand Down Expand Up @@ -68,12 +73,18 @@ type protobufPackage struct {
// A list of types to filter to; if not specified all types will be included.
FilterTypes map[types.Name]struct{}

// If true, omit any gogoprotobuf extensions not defined as types.
OmitGogo bool

// A list of field types that will be excluded from the output struct
OmitFieldTypes map[types.Name]struct{}

// A list of names that this package exports
LocalNames map[string]struct{}

// A list of struct tags to generate onto named struct fields
StructTags map[string]map[string]string

// An import tracker for this package
Imports *ImportTracker
}
Expand Down Expand Up @@ -127,6 +138,43 @@ func (p *protobufPackage) HasGoType(name string) bool {
return ok
}

func (p *protobufPackage) ExtractGeneratedType(t *ast.TypeSpec) bool {
if !p.HasGoType(t.Name.Name) {
return false
}

switch s := t.Type.(type) {
case *ast.StructType:
for i, f := range s.Fields.List {
if len(f.Tag.Value) == 0 {
continue
}
tag := strings.Trim(f.Tag.Value, "`")
protobufTag := reflect.StructTag(tag).Get("protobuf")
if len(protobufTag) == 0 {
continue
}
if len(f.Names) > 1 {
log.Printf("WARNING: struct %s field %d %s: defined multiple names but single protobuf tag", t.Name.Name, i, f.Names[0].Name)
// TODO hard error?
}
if p.StructTags == nil {
p.StructTags = make(map[string]map[string]string)
}
m := p.StructTags[t.Name.Name]
if m == nil {
m = make(map[string]string)
p.StructTags[t.Name.Name] = m
}
m[f.Names[0].Name] = tag
}
default:
log.Printf("WARNING: unexpected Go AST type definition: %#v", t)
}

return true
}

func (p *protobufPackage) Generators(c *generator.Context) []generator.Generator {
generators := []generator.Generator{}

Expand All @@ -140,6 +188,7 @@ func (p *protobufPackage) Generators(c *generator.Context) []generator.Generator
localGoPackage: types.Name{Package: p.PackagePath, Name: p.GoPackageName()},
imports: p.Imports,
generateAll: p.GenerateAll,
omitGogo: p.OmitGogo,
omitFieldTypes: p.OmitFieldTypes,
})
return generators
Expand Down
Loading

0 comments on commit 14a3aaf

Please sign in to comment.