Skip to content

Commit

Permalink
generate fixes and improvements (kubeflow#515)
Browse files Browse the repository at this point in the history
  • Loading branch information
@k8s-ci-robot
Kam Kasravi authored and k8s-ci-robot committed Oct 14, 2019
1 parent 3cc751e commit 9744780
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 70 deletions.
82 changes: 41 additions & 41 deletions hack/gen-test-target.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,16 @@ gen-target-start() {
echo 'package tests_test'
echo ''
echo 'import ('
echo ' "sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct"'
echo ' "sigs.k8s.io/kustomize/v3/k8sdeps/transformer"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/fs"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/loader"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/plugins"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/resmap"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/resource"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/target"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/validators"'
echo ' "testing"'
echo ' "sigs.k8s.io/kustomize/v3/k8sdeps/kunstruct"'
echo ' "sigs.k8s.io/kustomize/v3/k8sdeps/transformer"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/fs"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/loader"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/plugins"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/resmap"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/resource"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/target"'
echo ' "sigs.k8s.io/kustomize/v3/pkg/validators"'
echo ' "testing"'
echo ')'
echo ''
echo 'func write'$target'(th *KustTestHarness) {'
Expand Down Expand Up @@ -81,7 +81,7 @@ gen-target() {
}

gen-target-base() {
echo ' th.writeK("'$kname'", `'
echo ' th.writeK("'$kname'", `'
cat $dir/$file | sed 'sx- ../../basex- '$basedir'x' | sed 's/`/`+"`"+`/g'
echo '`)'
}
Expand All @@ -91,7 +91,7 @@ gen-target-kustomization() {
fname=/manifests${dir#*/manifests}
kname=${fname%/kustomization.yaml}
#echo 'gen-target-kustomization file='$file' dir='$dir' fname='$fname' kname='$kname
echo ' th.writeK("'$kname'", `'
echo ' th.writeK("'$kname'", `'
cat $dir/$file | sed 's/`/`+"`"+`/g'
echo '`)'
if [[ $(get-target-dirname $dir) != "base" ]]; then
Expand All @@ -108,7 +108,7 @@ gen-target-resource() {
if (( $# == 3 )); then
echooptions=$3
fi
echo $echooptions ' th.writeF("'$fname'", `'
echo $echooptions ' th.writeF("'$fname'", `'
cat $dir/$file | sed 's/`/`+"`"+`/g'
echo '`)'
}
Expand All @@ -123,34 +123,34 @@ gen-test-case() {
gen-target $1
echo ''
echo 'func Test'$target'(t *testing.T) {'
echo ' th := NewKustTestHarness(t, "'$fname'")'
echo ' write'$target'(th)'
echo ' m, err := th.makeKustTarget().MakeCustomizedResMap()'
echo ' if err != nil {'
echo ' t.Fatalf("Err: %v", err)'
echo ' }'
echo ' expected, err := m.AsYaml()'
echo ' if err != nil {'
echo ' t.Fatalf("Err: %v", err)'
echo ' }'
echo ' targetPath := "'$targetPath'"'
echo ' fsys := fs.MakeRealFS()'
echo ' lrc := loader.RestrictionRootOnly'
echo ' _loader, loaderErr := loader.NewLoader(lrc, validators.MakeFakeValidator(), targetPath, fsys)'
echo ' if loaderErr != nil {'
echo ' t.Fatalf("could not load kustomize loader: %v", loaderErr)'
echo ' }'
echo ' rf := resmap.NewFactory(resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()), transformer.NewFactoryImpl())'
echo ' pc := plugins.DefaultPluginConfig()'
echo ' kt, err := target.NewKustTarget(_loader, rf, transformer.NewFactoryImpl(), plugins.NewLoader(pc, rf))'
echo ' if err != nil {'
echo ' th.t.Fatalf("Unexpected construction error %v", err)'
echo ' }'
echo ' actual, err := kt.MakeCustomizedResMap()'
echo ' if err != nil {'
echo ' t.Fatalf("Err: %v", err)'
echo ' }'
echo ' th.assertActualEqualsExpected(actual, string(expected))'
echo ' th := NewKustTestHarness(t, "'$fname'")'
echo ' write'$target'(th)'
echo ' m, err := th.makeKustTarget().MakeCustomizedResMap()'
echo ' if err != nil {'
echo ' t.Fatalf("Err: %v", err)'
echo ' }'
echo ' expected, err := m.AsYaml()'
echo ' if err != nil {'
echo ' t.Fatalf("Err: %v", err)'
echo ' }'
echo ' targetPath := "'$targetPath'"'
echo ' fsys := fs.MakeRealFS()'
echo ' lrc := loader.RestrictionRootOnly'
echo ' _loader, loaderErr := loader.NewLoader(lrc, validators.MakeFakeValidator(), targetPath, fsys)'
echo ' if loaderErr != nil {'
echo ' t.Fatalf("could not load kustomize loader: %v", loaderErr)'
echo ' }'
echo ' rf := resmap.NewFactory(resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()), transformer.NewFactoryImpl())'
echo ' pc := plugins.DefaultPluginConfig()'
echo ' kt, err := target.NewKustTarget(_loader, rf, transformer.NewFactoryImpl(), plugins.NewLoader(pc, rf))'
echo ' if err != nil {'
echo ' th.t.Fatalf("Unexpected construction error %v", err)'
echo ' }'
echo ' actual, err := kt.MakeCustomizedResMap()'
echo ' if err != nil {'
echo ' t.Fatalf("Err: %v", err)'
echo ' }'
echo ' th.assertActualEqualsExpected(actual, string(expected))'
echo '}'
}

Expand Down
138 changes: 110 additions & 28 deletions hack/gen-test-targets.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,41 +6,109 @@
# The script is based on kusttestharness_test.go from kubernetes-sigs/pkg/kusttest/kusttestharness.go
#
add_app=''
all=false
changed=false
dry_run=false

if [[ $(basename $PWD) != "manifests" ]]; then
echo "must be at manifests root directory to run $0"
exit 1
fi

if [[ ! -f hack/utils.sh ]]; then
echo "$PWD/hack/utils.sh doesn't exist"
exit 1
fi

source hack/utils.sh

usage ()
usage()
{
echo -e "Usage: $0 [OPTIONS] <directory>\n"\
echo -e "Usage: $0 [OPTIONS] [<directory>]\n"\
'OPTIONS:\n'\
' -h | --help \n'\
' | --add-app <name=version>\n'\
' | --dry-run\n'
' | --add-app <name=version> <directory>\n'\
' -a | --all\n'\
' -c | --changed-only\n'\
' -d | --dry-run'
}

usage-extended()
{
echo ''
echo 'Examples:'
echo '1. Generate all unit tests.'
echo ' $ '$0' --all'
echo ' generating bootstrap-overlays-application_test.go from manifests/admission-webhook/bootstrap/overlays/application'
echo ' generating bootstrap-base_test.go from manifests/admission-webhook/bootstrap/base'
echo ' generating webhook-overlays-application_test.go from manifests/admission-webhook/webhook/overlays/application'
echo ' generating webhook-base_test.go from manifests/admission-webhook/webhook/base'
echo ' ...'
echo '2. Generate an application overlay for spartakus.'
echo ' $ '$0' --add-app spartakus=v0.7.0 common/spartakus'
echo ' mkdir -p common/spartakus/overlays/application'
echo ' git add common/spartakus/overlays/application'
echo ' generate common/spartakus/overlays/application'
echo '3. Generate unit tests for just the changed resources on the current branch.'
echo ' $ '$0' --changed-only'
echo ' generating webhook-overlays-application_test.go from manifests/admission-webhook/webhook/overlays/application'
echo '4. Show what unit tests would be generated for just the changed resources on the current branch.'
echo ' $ '$0' --changed-only --dry-run'
echo ' generating webhook-overlays-application_test.go from manifests/admission-webhook/webhook/overlays/application'
}

lsbranchname()
{
git branch|grep '^*'|awk '{print $2}'
}

findremotebranch()
{
git ls-remote --heads $(git config remote.origin.url) $(lsbranchname) | wc -l
}

findcommand()
{
# find * -type d -exec sh -c '(ls -p "{}"|grep />/dev/null)||echo "{}"' \; | egrep -v 'doc|tests|hack|plugins'
_findcommand()
{
local branch=$1
for i in $(git diff --name-only origin/${branch}..upstream/master | egrep -v 'doc|tests|hack|plugins');do
if [[ -f $i ]]; then
echo $(dirname $i);
fi
done
local branch i
case "$all" in
"false")
branch=$(lsbranchname)
for i in $(git diff --name-only origin/${branch}..upstream/master | egrep -v 'doc|tests|hack|plugins');do
if [[ -f $i ]]; then
echo $(dirname $i);
fi
done
;;
"true")
for i in $(find * -type d -exec sh -c '(ls -p "{}"|grep />/dev/null)||echo "{}"' \; | egrep -v 'doc|tests|hack|plugins|overlays'); do
if [[ -d $i ]]; then
echo $(dirname $i);
fi
done
;;
*)
echo "Unknown arguments: $@"
exit 1
;;
esac
}
_findcommand $(git branch|grep '^*'|awk '{print $2}') | sort | uniq
! $all && (( $(findremotebranch) == 0 )) && \
echo "Branch: origin/$(lsbranchname) doesn't exist, push local changes first" >&2 && exit 1
_findcommand | sort | uniq
}

addapp()
{
local app=${1%=*} version=${1#*=} appdir=${2}/overlays/application
local app=${1%=*} version=${1#*=} appdir=${2}/overlays/application cmd
#echo 'appdir='$appdir' app='$app' version='$version
if [[ ! -d $appdir ]]; then
mkdir -p $appdir
cmd="mkdir -p $appdir"
$dry_run && cmd='echo '$cmd
eval $cmd
fi
cat << KUSTOMIZATION > ${appdir}/kustomization.yaml
$dry_run || cat << KUSTOMIZATION > ${appdir}/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
bases:
Expand All @@ -56,7 +124,7 @@ commonLabels:
app.kubernetes.io/version: $version
KUSTOMIZATION

cat << APPLICATION > ${appdir}/application.yaml
$dry_run || cat << APPLICATION > ${appdir}/application.yaml
apiVersion: app.k8s.io/v1beta1
kind: Application
metadata:
Expand Down Expand Up @@ -89,7 +157,10 @@ spec:
url: ""
addOwnerRef: true
APPLICATION
git add $appdir

cmd="git add $appdir"
$dry_run && cmd='echo '$cmd
eval $cmd
}

generate()
Expand All @@ -114,11 +185,7 @@ generate()

generate-all()
{
if [[ $(basename $PWD) != "manifests" ]]; then
echo "must be at manifests root directory to run $0"
exit 1
fi
EXCLUDE_DIRS=( "kfdef" "gatekeeper" "gcp/deployment_manager_configs" "aws/infra_configs" )
EXCLUDE_DIRS=( "kfdef" "gatekeeper" "gcp/deployment_manager_configs" "aws/infra_configs" "." )
source hack/utils.sh
for i in $(findcommand); do
exclude=false
Expand All @@ -140,16 +207,28 @@ generate-all()
while :
do
case "$1" in
-h | --help)
-h)
usage
exit 0
;;
--help)
usage && usage-extended
exit 0
;;
--add-app)
shift
add_app=$1
shift
;;
--dry-run)
-a | --all)
all=true
shift
;;
-c | --changed-only)
changed=true
shift
;;
-d | --dry-run)
dry_run=true
shift
;;
Expand All @@ -165,19 +244,22 @@ done

case $# in
0)
generate-all
( $all || $changed ) && generate-all || \
( echo -e 'Either --all or --changed-only or directory required\n' && usage && exit 1 )
;;
1)
if (( $# == 1 )); then
cmd="generate $1"
if [[ -n $add_app ]]; then
addapp $add_app $1
cmd="generate $1/overlays/application"
fi
generate $1
$dry_run && cmd='echo '$cmd
eval $cmd
fi
;;
*)
echo "unknown arguments $@"
usage
echo "Unknown arguments: $@"
exit 1
;;
esac
6 changes: 5 additions & 1 deletion tests/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,11 @@ export GO = go
all: test

generate: clean
@cd .. && hack/gen-test-targets.sh && [ -n "$(EXCLUDE)" ] && rm -f tests/$(EXCLUDE) || echo done
@cd .. && hack/gen-test-targets.sh --all && [ -n "$(EXCLUDE)" ] && rm -f tests/$(EXCLUDE) || echo done
$(GO) fmt ./...

generate-changed-only:
@cd .. && hack/gen-test-targets.sh -c && [ -n "$(EXCLUDE)" ] && rm -f tests/$(EXCLUDE) || echo done
$(GO) fmt ./...

modules:
Expand Down

0 comments on commit 9744780

Please sign in to comment.