Skip to content

Commit

Permalink
Merge pull request kubernetes#19426 from smarterclayton/rewrite_tags
Browse files Browse the repository at this point in the history 
Auto commit by PR queue bot
  • Loading branch information
k8s-merge-robot committed Jan 26, 2016
2 parents 6f1ec1b + 14a3aaf commit 3254df3
Show file tree
Hide file tree
Showing 6 changed files with 403 additions and 53 deletions.
90 changes: 70 additions & 20 deletions cmd/libs/go2idl/go-to-protobuf/protobuf/cmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type Generator struct {
Conditional string
Clean bool
OnlyIDL bool
KeepGogoproto bool
SkipGeneratedRewrite bool
DropEmbeddedFields string
}
Expand Down Expand Up @@ -77,6 +78,7 @@ func (g *Generator) BindFlags(flag *flag.FlagSet) {
flag.StringVar(&g.Conditional, "conditional", g.Conditional, "An optional Golang build tag condition to add to the generated Go code")
flag.BoolVar(&g.Clean, "clean", g.Clean, "If true, remove all generated files for the specified Packages.")
flag.BoolVar(&g.OnlyIDL, "only-idl", g.OnlyIDL, "If true, only generate the IDL for each package.")
flag.BoolVar(&g.KeepGogoproto, "keep-gogoproto", g.KeepGogoproto, "If true, the generated IDL will contain gogoprotobuf extensions which are normally removed")
flag.BoolVar(&g.SkipGeneratedRewrite, "skip-generated-rewrite", g.SkipGeneratedRewrite, "If true, skip fixing up the generated.pb.go file (debugging only).")
flag.StringVar(&g.DropEmbeddedFields, "drop-embedded-fields", g.DropEmbeddedFields, "Comma-delimited list of embedded Go types to omit from generated protobufs")
}
Expand Down Expand Up @@ -206,8 +208,11 @@ func Run(g *Generator) {

for _, outputPackage := range outputPackages {
p := outputPackage.(*protobufPackage)

path := filepath.Join(g.OutputBase, p.ImportPath())
outputPath := filepath.Join(g.OutputBase, p.OutputPath())

// generate the gogoprotobuf protoc
cmd := exec.Command("protoc", append(args, path)...)
out, err := cmd.CombinedOutput()
if len(out) > 0 {
Expand All @@ -217,29 +222,74 @@ func Run(g *Generator) {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err)
}
if !g.SkipGeneratedRewrite {
if err := RewriteGeneratedGogoProtobufFile(outputPath, p.GoPackageName(), p.HasGoType, buf.Bytes()); err != nil {
log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
}

cmd := exec.Command("goimports", "-w", outputPath)
out, err := cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
}
if g.SkipGeneratedRewrite {
continue
}

// alter the generated protobuf file to remove the generated types (but leave the serializers) and rewrite the
// package statement to match the desired package name
if err := RewriteGeneratedGogoProtobufFile(outputPath, p.GoPackageName(), p.ExtractGeneratedType, buf.Bytes()); err != nil {
log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
}

// sort imports
cmd = exec.Command("goimports", "-w", outputPath)
out, err = cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
}

// format and simplify the generated file
cmd = exec.Command("gofmt", "-s", "-w", outputPath)
out, err = cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to apply gofmt for %s: %v", p.PackageName, err)
}
}

if g.SkipGeneratedRewrite {
return
}

if !g.KeepGogoproto {
// generate, but do so without gogoprotobuf extensions
for _, outputPackage := range outputPackages {
p := outputPackage.(*protobufPackage)
p.OmitGogo = true
}
if err := c.ExecutePackages(g.OutputBase, outputPackages); err != nil {
log.Fatalf("Failed executing generator: %v", err)
}
}

for _, outputPackage := range outputPackages {
p := outputPackage.(*protobufPackage)

if len(p.StructTags) == 0 {
continue
}

pattern := filepath.Join(g.OutputBase, p.PackagePath, "*.go")
files, err := filepath.Glob(pattern)
if err != nil {
log.Fatalf("Can't glob pattern %q: %v", pattern, err)
}

cmd = exec.Command("gofmt", "-s", "-w", outputPath)
out, err = cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
for _, s := range files {
if strings.HasSuffix(s, "_test.go") {
continue
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
if err := RewriteTypesWithProtobufStructTags(s, p.StructTags); err != nil {
log.Fatalf("Unable to rewrite with struct tags %s: %v", s, err)
}
}
}
Expand Down
83 changes: 54 additions & 29 deletions cmd/libs/go2idl/go-to-protobuf/protobuf/generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,16 @@ type genProtoIDL struct {
imports *ImportTracker

generateAll bool
omitGogo bool
omitFieldTypes map[types.Name]struct{}
}

func (g *genProtoIDL) PackageVars(c *generator.Context) []string {
if g.omitGogo {
return []string{
fmt.Sprintf("option go_package = %q;", g.localGoPackage.Name),
}
}
return []string{
"option (gogoproto.marshaler_all) = true;",
"option (gogoproto.sizer_all) = true;",
Expand Down Expand Up @@ -117,7 +123,15 @@ func isProtoable(seen map[*types.Type]bool, t *types.Type) bool {
}

func (g *genProtoIDL) Imports(c *generator.Context) (imports []string) {
return g.imports.ImportLines()
lines := []string{}
// TODO: this could be expressed more cleanly
for _, line := range g.imports.ImportLines() {
if g.omitGogo && line == "github.com/gogo/protobuf/gogoproto/gogo.proto" {
continue
}
lines = append(lines, line)
}
return lines
}

// GenerateType makes the body of a file implementing a set for type t.
Expand All @@ -130,7 +144,9 @@ func (g *genProtoIDL) GenerateType(c *generator.Context, t *types.Type, w io.Wri

localGoPackage: g.localGoPackage.Package,
},
localPackage: g.localPackage,
localPackage: g.localPackage,

omitGogo: g.omitGogo,
omitFieldTypes: g.omitFieldTypes,

t: t,
Expand Down Expand Up @@ -201,6 +217,7 @@ func (p protobufLocator) ProtoTypeFor(t *types.Type) (*types.Type, error) {
type bodyGen struct {
locator ProtobufLocator
localPackage types.Name
omitGogo bool
omitFieldTypes map[types.Name]struct{}

t *types.Type
Expand Down Expand Up @@ -228,14 +245,18 @@ func (b bodyGen) doStruct(sw *generator.SnippetWriter) error {
switch key {
case "marshal":
if v == "false" {
options = append(options,
"(gogoproto.marshaler) = false",
"(gogoproto.unmarshaler) = false",
"(gogoproto.sizer) = false",
)
if !b.omitGogo {
options = append(options,
"(gogoproto.marshaler) = false",
"(gogoproto.unmarshaler) = false",
"(gogoproto.sizer) = false",
)
}
}
default:
options = append(options, fmt.Sprintf("%s = %s", key, v))
if !b.omitGogo || !strings.HasPrefix(key, "(gogoproto.") {
options = append(options, fmt.Sprintf("%s = %s", key, v))
}
}
case k == "protobuf.embed":
fields = []protoField{
Expand Down Expand Up @@ -289,14 +310,19 @@ func (b bodyGen) doStruct(sw *generator.SnippetWriter) error {
}
sw.Do(`$.Type|local$ $.Name$ = $.Tag$`, field)
if len(field.Extras) > 0 {
fmt.Fprintf(out, " [")
extras := []string{}
for k, v := range field.Extras {
if b.omitGogo && strings.HasPrefix(k, "(gogoproto.") {
continue
}
extras = append(extras, fmt.Sprintf("%s = %s", k, v))
}
sort.Sort(sort.StringSlice(extras))
fmt.Fprint(out, strings.Join(extras, ", "))
fmt.Fprintf(out, "]")
if len(extras) > 0 {
fmt.Fprintf(out, " [")
fmt.Fprint(out, strings.Join(extras, ", "))
fmt.Fprintf(out, "]")
}
}
fmt.Fprintf(out, ";\n")
if i != len(fields)-1 {
Expand Down Expand Up @@ -459,24 +485,19 @@ func protobufTagToField(tag string, field *protoField, m types.Member, t *types.
Kind: typesKindProtobuf,
}
} else {
field.Type = &types.Type{
Name: types.Name{
Name: parts[0],
Package: localPackage.Package,
Path: localPackage.Path,
},
Kind: typesKindProtobuf,
switch parts[0] {
case "varint", "bytes", "fixed64":
default:
field.Type = &types.Type{
Name: types.Name{
Name: parts[0],
Package: localPackage.Package,
Path: localPackage.Path,
},
Kind: typesKindProtobuf,
}
}
}
switch parts[2] {
case "rep":
field.Repeated = true
case "opt":
field.Optional = true
case "req":
default:
return fmt.Errorf("member %q of %q malformed 'protobuf' tag, field mode is %q not recognized\n", m.Name, t.Name, parts[2])
}
field.OptionalSet = true

protoExtra := make(map[string]string)
Expand All @@ -485,7 +506,11 @@ func protobufTagToField(tag string, field *protoField, m types.Member, t *types.
if len(parts) != 2 {
return fmt.Errorf("member %q of %q malformed 'protobuf' tag, tag %d should be key=value, got %q\n", m.Name, t.Name, i+4, extra)
}
protoExtra[parts[0]] = parts[1]
switch parts[0] {
case "casttype":
parts[0] = fmt.Sprintf("(gogoproto.%s)", parts[0])
protoExtra[parts[0]] = parts[1]
}
}

field.Extras = protoExtra
Expand Down Expand Up @@ -526,7 +551,7 @@ func membersToFields(locator ProtobufLocator, t *types.Type, localPackage types.
if len(field.Name) == 0 && len(parts[0]) != 0 {
field.Name = parts[0]
}
if field.Name == "-" {
if field.Tag == -1 && field.Name == "-" {
continue
}
}
Expand Down
49 changes: 49 additions & 0 deletions cmd/libs/go2idl/go-to-protobuf/protobuf/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,13 @@ package protobuf

import (
"fmt"
"log"
"os"
"path/filepath"
"reflect"
"strings"

"k8s.io/kubernetes/third_party/golang/go/ast"

"k8s.io/kubernetes/cmd/libs/go2idl/generator"
"k8s.io/kubernetes/cmd/libs/go2idl/types"
Expand Down Expand Up @@ -68,12 +73,18 @@ type protobufPackage struct {
// A list of types to filter to; if not specified all types will be included.
FilterTypes map[types.Name]struct{}

// If true, omit any gogoprotobuf extensions not defined as types.
OmitGogo bool

// A list of field types that will be excluded from the output struct
OmitFieldTypes map[types.Name]struct{}

// A list of names that this package exports
LocalNames map[string]struct{}

// A list of struct tags to generate onto named struct fields
StructTags map[string]map[string]string

// An import tracker for this package
Imports *ImportTracker
}
Expand Down Expand Up @@ -127,6 +138,43 @@ func (p *protobufPackage) HasGoType(name string) bool {
return ok
}

func (p *protobufPackage) ExtractGeneratedType(t *ast.TypeSpec) bool {
if !p.HasGoType(t.Name.Name) {
return false
}

switch s := t.Type.(type) {
case *ast.StructType:
for i, f := range s.Fields.List {
if len(f.Tag.Value) == 0 {
continue
}
tag := strings.Trim(f.Tag.Value, "`")
protobufTag := reflect.StructTag(tag).Get("protobuf")
if len(protobufTag) == 0 {
continue
}
if len(f.Names) > 1 {
log.Printf("WARNING: struct %s field %d %s: defined multiple names but single protobuf tag", t.Name.Name, i, f.Names[0].Name)
// TODO hard error?
}
if p.StructTags == nil {
p.StructTags = make(map[string]map[string]string)
}
m := p.StructTags[t.Name.Name]
if m == nil {
m = make(map[string]string)
p.StructTags[t.Name.Name] = m
}
m[f.Names[0].Name] = tag
}
default:
log.Printf("WARNING: unexpected Go AST type definition: %#v", t)
}

return true
}

func (p *protobufPackage) Generators(c *generator.Context) []generator.Generator {
generators := []generator.Generator{}

Expand All @@ -140,6 +188,7 @@ func (p *protobufPackage) Generators(c *generator.Context) []generator.Generator
localGoPackage: types.Name{Package: p.PackagePath, Name: p.GoPackageName()},
imports: p.Imports,
generateAll: p.GenerateAll,
omitGogo: p.OmitGogo,
omitFieldTypes: p.OmitFieldTypes,
})
return generators
Expand Down
Loading

0 comments on commit 3254df3

Please sign in to comment.