Skip to content

Commit

Permalink
Improve conversion to support multiple packages
Browse files Browse the repository at this point in the history 
OpenShift uses multiple API packages (types are split) which
Kube will also eventually have as we introduce more plugins.
These changes make the generators able to handle importing different
API object packages into a single generator function.
  • Loading branch information
@smarterclayton
smarterclayton committed Jul 29, 2015
1 parent fa6ee98 commit 732647e
Show file tree
Hide file tree
Showing 5 changed files with 325 additions and 74 deletions.
13 changes: 11 additions & 2 deletions cmd/genconversion/conversion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,16 @@ limitations under the License.
package main

import (
"fmt"
"io"
"os"
"path"
"runtime"

"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1"
pkg_runtime "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

"github.com/golang/glog"
flag "github.com/spf13/pflag"
Expand All @@ -50,18 +53,24 @@ func main() {
funcOut = file
}

generator := pkg_runtime.NewConversionGenerator(api.Scheme.Raw())
generator := pkg_runtime.NewConversionGenerator(api.Scheme.Raw(), path.Join("github.com/GoogleCloudPlatform/kubernetes/pkg/api", *version))
apiShort := generator.AddImport("github.com/GoogleCloudPlatform/kubernetes/pkg/api")
generator.AddImport("github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource")
// TODO(wojtek-t): Change the overwrites to a flag.
generator.OverwritePackage(*version, "")
for _, knownType := range api.Scheme.KnownTypes(*version) {
if err := generator.GenerateConversionsForType(*version, knownType); err != nil {
glog.Errorf("error while generating conversion functions for %v: %v", knownType, err)
}
}
generator.RepackImports(util.NewStringSet())
if err := generator.WriteImports(funcOut); err != nil {
glog.Fatalf("error while writing imports: %v", err)
}
if err := generator.WriteConversionFunctions(funcOut); err != nil {
glog.Fatalf("Error while writing conversion functions: %v", err)
}
if err := generator.RegisterConversionFunctions(funcOut); err != nil {
if err := generator.RegisterConversionFunctions(funcOut, fmt.Sprintf("%s.Scheme", apiShort)); err != nil {
glog.Fatalf("Error while writing conversion functions: %v", err)
}
}
13 changes: 10 additions & 3 deletions cmd/gendeepcopy/deep_copy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@ package main
import (
"io"
"os"
"path"
"runtime"
"strings"

"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1"
pkg_runtime "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

"github.com/golang/glog"
flag "github.com/spf13/pflag"
Expand Down Expand Up @@ -53,10 +55,14 @@ func main() {
}

knownVersion := *version
registerTo := "api.Scheme"
if knownVersion == "api" {
knownVersion = api.Scheme.Raw().InternalVersion
registerTo = "Scheme"
}
generator := pkg_runtime.NewDeepCopyGenerator(api.Scheme.Raw())
pkgPath := path.Join("github.com/GoogleCloudPlatform/kubernetes/pkg/api", knownVersion)
generator := pkg_runtime.NewDeepCopyGenerator(api.Scheme.Raw(), pkgPath, util.NewStringSet("github.com/GoogleCloudPlatform/kubernetes"))
generator.AddImport("github.com/GoogleCloudPlatform/kubernetes/pkg/api")

for _, overwrite := range strings.Split(*overwrites, ",") {
vals := strings.Split(overwrite, "=")
Expand All @@ -67,13 +73,14 @@ func main() {
glog.Errorf("error while generating deep copy functions for %v: %v", knownType, err)
}
}
if err := generator.WriteImports(funcOut, *version); err != nil {
generator.RepackImports()
if err := generator.WriteImports(funcOut); err != nil {
glog.Fatalf("error while writing imports: %v", err)
}
if err := generator.WriteDeepCopyFunctions(funcOut); err != nil {
glog.Fatalf("error while writing deep copy functions: %v", err)
}
if err := generator.RegisterDeepCopyFunctions(funcOut, *version); err != nil {
if err := generator.RegisterDeepCopyFunctions(funcOut, registerTo); err != nil {
glog.Fatalf("error while registering deep copy functions: %v", err)
}
}
8 changes: 0 additions & 8 deletions hack/update-generated-conversions.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,6 @@ function generate_version() {
cat >> $TMPFILE <<EOF
package ${version}
import (
"reflect"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource"
"github.com/GoogleCloudPlatform/kubernetes/pkg/conversion"
)
// AUTO-GENERATED FUNCTIONS START HERE
EOF

Expand Down
177 changes: 154 additions & 23 deletions pkg/runtime/conversion_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,40 +19,69 @@ package runtime
import (
"fmt"
"io"
"path"
"reflect"
"sort"
"strings"

"github.com/GoogleCloudPlatform/kubernetes/pkg/conversion"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)

type ConversionGenerator interface {
GenerateConversionsForType(version string, reflection reflect.Type) error
WriteConversionFunctions(w io.Writer) error
RegisterConversionFunctions(w io.Writer) error
RegisterConversionFunctions(w io.Writer, pkg string) error
AddImport(pkg string) string
RepackImports(exclude util.StringSet)
WriteImports(w io.Writer) error
OverwritePackage(pkg, overwrite string)
AssumePrivateConversions()
}

func NewConversionGenerator(scheme *conversion.Scheme) ConversionGenerator {
return &conversionGenerator{
func NewConversionGenerator(scheme *conversion.Scheme, targetPkg string) ConversionGenerator {
g := &conversionGenerator{
scheme: scheme,
targetPkg: targetPkg,
convertibles: make(map[reflect.Type]reflect.Type),
pkgOverwrites: make(map[string]string),
imports: make(map[string]string),
shortImports: make(map[string]string),
}
g.targetPackage(targetPkg)
g.AddImport("reflect")
g.AddImport("github.com/GoogleCloudPlatform/kubernetes/pkg/conversion")
return g
}

var complexTypes []reflect.Kind = []reflect.Kind{reflect.Map, reflect.Ptr, reflect.Slice, reflect.Interface, reflect.Struct}

type conversionGenerator struct {
scheme *conversion.Scheme
targetPkg string
convertibles map[reflect.Type]reflect.Type
// If pkgOverwrites is set for a given package name, that package name
// will be replaced while writing conversion function. If empty, package
// name will be omitted.
pkgOverwrites map[string]string
// map of package names to shortname
imports map[string]string
// map of short names to package names
shortImports map[string]string

// A buffer that is used for storing lines that needs to be written.
linesToPrint []string

// if true, we assume conversions on the scheme are not available to us in the current package
assumePrivateConversions bool
}

func (g *conversionGenerator) AssumePrivateConversions() {
g.assumePrivateConversions = true
}

func (g *conversionGenerator) AddImport(pkg string) string {
return g.addImportByPath(pkg)
}

func (g *conversionGenerator) GenerateConversionsForType(version string, reflection reflect.Type) error {
Expand Down Expand Up @@ -94,6 +123,10 @@ func (g *conversionGenerator) generateConversionsBetween(inType, outType reflect
}
return fmt.Errorf("cannot convert types of different kinds: %v %v", inType, outType)
}

g.addImportByPath(inType.PkgPath())
g.addImportByPath(outType.PkgPath())

// We should be able to generate conversions both sides.
switch inType.Kind() {
case reflect.Map:
Expand Down Expand Up @@ -155,11 +188,15 @@ func isComplexType(reflection reflect.Type) bool {
func (g *conversionGenerator) generateConversionsForMap(inType, outType reflect.Type) error {
inKey := inType.Key()
outKey := outType.Key()
g.addImportByPath(inKey.PkgPath())
g.addImportByPath(outKey.PkgPath())
if err := g.generateConversionsBetween(inKey, outKey); err != nil {
return err
}
inValue := inType.Elem()
outValue := outType.Elem()
g.addImportByPath(inKey.PkgPath())
g.addImportByPath(outKey.PkgPath())
if err := g.generateConversionsBetween(inValue, outValue); err != nil {
return err
}
Expand Down Expand Up @@ -238,6 +275,54 @@ func (s byName) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}

func (g *conversionGenerator) targetPackage(pkg string) {
g.imports[pkg] = ""
g.shortImports[""] = pkg
}

func (g *conversionGenerator) RepackImports(exclude util.StringSet) {
var packages []string
for key := range g.imports {
packages = append(packages, key)
}
sort.Strings(packages)
g.imports = make(map[string]string)
g.shortImports = make(map[string]string)
g.targetPackage(g.targetPkg)
for _, pkg := range packages {
if !exclude.Has(pkg) {
g.addImportByPath(pkg)
}
}
}

func (g *conversionGenerator) WriteImports(w io.Writer) error {
var packages []string
for key := range g.imports {
packages = append(packages, key)
}
sort.Strings(packages)

buffer := newBuffer()
indent := 0
buffer.addLine("import (\n", indent)
for _, importPkg := range packages {
if len(importPkg) == 0 {
continue
}
if len(g.imports[importPkg]) == 0 {
continue
}
buffer.addLine(fmt.Sprintf("%s \"%s\"\n", g.imports[importPkg], importPkg), indent+1)
}
buffer.addLine(")\n", indent)
buffer.addLine("\n", indent)
if err := buffer.flushLines(w); err != nil {
return err
}
return nil
}

func (g *conversionGenerator) WriteConversionFunctions(w io.Writer) error {
// It's desired to print conversion functions always in the same order
// (e.g. for better tracking of what has really been added).
Expand Down Expand Up @@ -265,9 +350,9 @@ func (g *conversionGenerator) WriteConversionFunctions(w io.Writer) error {
return nil
}

func (g *conversionGenerator) writeRegisterHeader(b *buffer, indent int) {
func (g *conversionGenerator) writeRegisterHeader(b *buffer, pkg string, indent int) {
b.addLine("func init() {\n", indent)
b.addLine("err := api.Scheme.AddGeneratedConversionFuncs(\n", indent+1)
b.addLine(fmt.Sprintf("err := %s.AddGeneratedConversionFuncs(\n", pkg), indent+1)
}

func (g *conversionGenerator) writeRegisterFooter(b *buffer, indent int) {
Expand All @@ -280,7 +365,7 @@ func (g *conversionGenerator) writeRegisterFooter(b *buffer, indent int) {
b.addLine("\n", indent)
}

func (g *conversionGenerator) RegisterConversionFunctions(w io.Writer) error {
func (g *conversionGenerator) RegisterConversionFunctions(w io.Writer, pkg string) error {
// Write conversion function names alphabetically ordered.
var names []string
for inType, outType := range g.convertibles {
Expand All @@ -290,7 +375,7 @@ func (g *conversionGenerator) RegisterConversionFunctions(w io.Writer) error {

buffer := newBuffer()
indent := 0
g.writeRegisterHeader(buffer, indent)
g.writeRegisterHeader(buffer, pkg, indent)
for _, name := range names {
buffer.addLine(fmt.Sprintf("%s,\n", name), indent+2)
}
Expand All @@ -301,32 +386,74 @@ func (g *conversionGenerator) RegisterConversionFunctions(w io.Writer) error {
return nil
}

func (g *conversionGenerator) addImportByPath(pkg string) string {
if name, ok := g.imports[pkg]; ok {
return name
}
name := path.Base(pkg)
if _, ok := g.shortImports[name]; !ok {
g.imports[pkg] = name
g.shortImports[name] = pkg
return name
}
if dirname := path.Base(path.Dir(pkg)); len(dirname) > 0 {
name = dirname + name
if _, ok := g.shortImports[name]; !ok {
g.imports[pkg] = name
g.shortImports[name] = pkg
return name
}
if subdirname := path.Base(path.Dir(path.Dir(pkg))); len(subdirname) > 0 {
name = subdirname + name
if _, ok := g.shortImports[name]; !ok {
g.imports[pkg] = name
g.shortImports[name] = pkg
return name
}
}
}
for i := 2; i < 100; i++ {
generatedName := fmt.Sprintf("%s%d", name, i)
if _, ok := g.shortImports[generatedName]; !ok {
g.imports[pkg] = generatedName
g.shortImports[generatedName] = pkg
return generatedName
}
}
panic(fmt.Sprintf("unable to find a unique name for the package path %q: %v", pkg, g.shortImports))
}

func (g *conversionGenerator) typeName(inType reflect.Type) string {
switch inType.Kind() {
case reflect.Map:
return fmt.Sprintf("map[%s]%s", g.typeName(inType.Key()), g.typeName(inType.Elem()))
case reflect.Slice:
return fmt.Sprintf("[]%s", g.typeName(inType.Elem()))
case reflect.Ptr:
return fmt.Sprintf("*%s", g.typeName(inType.Elem()))
case reflect.Map:
if len(inType.Name()) == 0 {
return fmt.Sprintf("map[%s]%s", g.typeName(inType.Key()), g.typeName(inType.Elem()))
}
fallthrough
default:
typeWithPkg := fmt.Sprintf("%s", inType)
slices := strings.Split(typeWithPkg, ".")
if len(slices) == 1 {
pkg, name := inType.PkgPath(), inType.Name()
if len(name) == 0 && inType.Kind() == reflect.Struct {
return "struct{}"
}
if len(pkg) == 0 {
// Default package.
return slices[0]
return name
}
if len(slices) == 2 {
pkg := slices[0]
if val, found := g.pkgOverwrites[pkg]; found {
pkg = val
}
if pkg != "" {
pkg = pkg + "."
}
return pkg + slices[1]
if val, found := g.pkgOverwrites[pkg]; found {
pkg = val
}
if len(pkg) == 0 {
return name
}
panic("Incorrect type name: " + typeWithPkg)
short := g.addImportByPath(pkg)
if len(short) > 0 {
return fmt.Sprintf("%s.%s", short, name)
}
return name
}
}

Expand Down Expand Up @@ -658,6 +785,10 @@ func (g *conversionGenerator) existsDedicatedConversionFunction(inType, outType
// unnamed. Thus we return false here.
return false
}
// TODO: no way to handle private conversions in different packages
if g.assumePrivateConversions {
return false
}
return g.scheme.Converter().HasConversionFunc(inType, outType)
}

Expand Down
Loading

0 comments on commit 732647e

Please sign in to comment.