Merge pull request kubernetes#13613 from eparis/rework-doc-generation

Rework automatic doc generation
gosuri · Sep 16, 2015 · ebc0b02 · ebc0b02
2 parents a1294e6 + 58d6b29
commit ebc0b02
Show file tree

Hide file tree

Showing 8 changed files with 153 additions and 146 deletions.
diff --git a/.generated_docs b/.generated_docs
@@ -0,0 +1,64 @@
+.generated_docs
+contrib/completions/bash/kubectl
+docs/man/man1/kubectl-annotate.1
+docs/man/man1/kubectl-api-versions.1
+docs/man/man1/kubectl-attach.1
+docs/man/man1/kubectl-cluster-info.1
+docs/man/man1/kubectl-config-set-cluster.1
+docs/man/man1/kubectl-config-set-context.1
+docs/man/man1/kubectl-config-set-credentials.1
+docs/man/man1/kubectl-config-set.1
+docs/man/man1/kubectl-config-unset.1
+docs/man/man1/kubectl-config-use-context.1
+docs/man/man1/kubectl-config-view.1
+docs/man/man1/kubectl-config.1
+docs/man/man1/kubectl-create.1
+docs/man/man1/kubectl-delete.1
+docs/man/man1/kubectl-describe.1
+docs/man/man1/kubectl-exec.1
+docs/man/man1/kubectl-expose.1
+docs/man/man1/kubectl-get.1
+docs/man/man1/kubectl-label.1
+docs/man/man1/kubectl-logs.1
+docs/man/man1/kubectl-namespace.1
+docs/man/man1/kubectl-patch.1
+docs/man/man1/kubectl-port-forward.1
+docs/man/man1/kubectl-proxy.1
+docs/man/man1/kubectl-replace.1
+docs/man/man1/kubectl-rolling-update.1
+docs/man/man1/kubectl-run.1
+docs/man/man1/kubectl-scale.1
+docs/man/man1/kubectl-stop.1
+docs/man/man1/kubectl-version.1
+docs/man/man1/kubectl.1
+docs/user-guide/kubectl/kubectl.md
+docs/user-guide/kubectl/kubectl_annotate.md
+docs/user-guide/kubectl/kubectl_api-versions.md
+docs/user-guide/kubectl/kubectl_attach.md
+docs/user-guide/kubectl/kubectl_cluster-info.md
+docs/user-guide/kubectl/kubectl_config.md
+docs/user-guide/kubectl/kubectl_config_set-cluster.md
+docs/user-guide/kubectl/kubectl_config_set-context.md
+docs/user-guide/kubectl/kubectl_config_set-credentials.md
+docs/user-guide/kubectl/kubectl_config_set.md
+docs/user-guide/kubectl/kubectl_config_unset.md
+docs/user-guide/kubectl/kubectl_config_use-context.md
+docs/user-guide/kubectl/kubectl_config_view.md
+docs/user-guide/kubectl/kubectl_create.md
+docs/user-guide/kubectl/kubectl_delete.md
+docs/user-guide/kubectl/kubectl_describe.md
+docs/user-guide/kubectl/kubectl_exec.md
+docs/user-guide/kubectl/kubectl_expose.md
+docs/user-guide/kubectl/kubectl_get.md
+docs/user-guide/kubectl/kubectl_label.md
+docs/user-guide/kubectl/kubectl_logs.md
+docs/user-guide/kubectl/kubectl_namespace.md
+docs/user-guide/kubectl/kubectl_patch.md
+docs/user-guide/kubectl/kubectl_port-forward.md
+docs/user-guide/kubectl/kubectl_proxy.md
+docs/user-guide/kubectl/kubectl_replace.md
+docs/user-guide/kubectl/kubectl_rolling-update.md
+docs/user-guide/kubectl/kubectl_run.md
+docs/user-guide/kubectl/kubectl_scale.md
+docs/user-guide/kubectl/kubectl_stop.md
+docs/user-guide/kubectl/kubectl_version.md
diff --git a/contrib/completions/bash/.files_generated b/contrib/completions/bash/.files_generated
diff --git a/docs/man/man1/.files_generated b/docs/man/man1/.files_generated
diff --git a/docs/user-guide/kubectl/.files_generated b/docs/user-guide/kubectl/.files_generated
diff --git a/hack/after-build/update-generated-docs.sh b/hack/after-build/update-generated-docs.sh
@@ -23,17 +23,23 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 
 kube::golang::setup_env
 
-# Find binary
-gendocs=$(kube::util::find-binary "gendocs")
-genman=$(kube::util::find-binary "genman")
-genbashcomp=$(kube::util::find-binary "genbashcomp")
-mungedocs=$(kube::util::find-binary "mungedocs")
+kube::util::ensure-temp-dir
+
+kube::util::gen-docs "${KUBE_TEMP}"
+
+# remove all of the old docs
+while read file; do
+  rm "${KUBE_ROOT}/${file}" 2>/dev/null || true
+done <"${KUBE_ROOT}/.generated_docs"
+
+# the shopt is so that we get .generated_docs from the glob.
+shopt -s dotglob
+cp -af "${KUBE_TEMP}"/* "${KUBE_ROOT}"
+shopt -u dotglob
 
-kube::util::gen-doc "${gendocs}" "${KUBE_ROOT}" "docs/user-guide/kubectl/" '###### Auto generated by spf13/cobra'
-kube::util::gen-doc "${genman}" "${KUBE_ROOT}" "docs/man/man1"
-kube::util::gen-doc "${genbashcomp}" "${KUBE_ROOT}" "contrib/completions/bash/"
 kube::util::gen-analytics "${KUBE_ROOT}"
 
+mungedocs=$(kube::util::find-binary "mungedocs")
 "${mungedocs}" "--root-dir=${KUBE_ROOT}/docs/" && ret=0 || ret=$?
 if [[ $ret -eq 1 ]]; then
   echo "${KUBE_ROOT}/docs/ requires manual changes.  See preceding errors."

diff --git a/hack/after-build/verify-generated-docs.sh b/hack/after-build/verify-generated-docs.sh
@@ -31,11 +31,6 @@ mungedocs=$(kube::util::find-binary "mungedocs")
 
 DOCROOT="${KUBE_ROOT}/docs/"
 EXAMPLEROOT="${KUBE_ROOT}/examples/"
-TMP_DOCROOT="${KUBE_ROOT}/_tmp/docs/"
-_tmp="${KUBE_ROOT}/_tmp"
-
-mkdir -p "${_tmp}"
-cp -a "${DOCROOT}" "${TMP_DOCROOT}"
 
 # mungedocs --verify can (and should) be run on the real docs, otherwise their
 # links will be distorted. --verify means that it will not make changes.
@@ -59,12 +54,15 @@ if [[ $ret -gt 1 ]]; then
   exit 1
 fi
 
-kube::util::gen-doc "${genman}" "${_tmp}" "docs/man/man1/"
-kube::util::gen-doc "${gendocs}" "${_tmp}" "docs/user-guide/kubectl/" '###### Auto generated by spf13/cobra'
 
-echo "diffing ${DOCROOT} against freshly generated docs"
-diff -Naupr "${DOCROOT}" "${TMP_DOCROOT}" && ret=0 || ret=$?
-rm -rf "${_tmp}"
+kube::util::ensure-temp-dir
+
+kube::util::gen-docs "${KUBE_TEMP}"
+diff -Naup "${KUBE_TEMP}/.generated_docs" "${KUBE_ROOT}/.generated_docs" || ret=1 || true
+while read file; do
+  diff -Naup "${KUBE_TEMP}/${file}" "${KUBE_ROOT}/${file}" || ret=1 || true
+done <"${KUBE_TEMP}/.generated_docs"
+
 needsanalytics=($(kube::util::gen-analytics "${KUBE_ROOT}" 1))
 if [[ ${#needsanalytics[@]} -ne 0 ]]; then
   echo -e "Some md files are missing ga-beacon analytics link:"
@@ -73,26 +71,9 @@ if [[ ${#needsanalytics[@]} -ne 0 ]]; then
 fi
 if [[ $ret -eq 0 ]]
 then
-  echo "${DOCROOT} up to date."
+  echo "Generated docs are up to date."
 else
-  echo "${DOCROOT} is out of date. Please run hack/update-generated-docs.sh"
+  echo "Generated docs are out of date. Please run hack/update-generated-docs.sh"
   exit 1
 fi
-
-COMPROOT="${KUBE_ROOT}/contrib/completions"
-TMP_COMPROOT="${KUBE_ROOT}/contrib/completions_tmp"
-cp -a "${COMPROOT}" "${TMP_COMPROOT}"
-kube::util::gen-doc "${genbashcomp}" "${TMP_COMPROOT}" "bash/"
-diff -Naupr "${COMPROOT}" "${TMP_COMPROOT}" && ret=0 || ret=$?
-rm -rf ${TMP_COMPROOT}
-if [ $ret -eq 0 ]
-then
-	echo "${COMPROOT} up to date."
-else
-	echo "${COMPROOT} is out of date. Please run hack/update-generated-docs.sh"
-	echo "If you did not make a change to kubectl or its dependencies,"
-	echo "run 'make clean' and retry this command."
-	exit 1
-fi
-
 # ex: ts=2 sw=2 et filetype=sh
diff --git a/hack/lib/util.sh b/hack/lib/util.sh
@@ -48,13 +48,44 @@ kube::util::wait_for_url() {
   return 1
 }
 
+# Example:  kube::util::trap_add 'echo "in trap DEBUG"' DEBUG
+# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal
+kube::util::trap_add() {
+  local trap_add_cmd
+  trap_add_cmd=$1
+  shift
+
+  for trap_add_name in "$@"; do
+    local existing_cmd
+    local new_cmd
+
+    # Grab the currently defined trap commands for this trap
+    existing_cmd=`trap -p "${trap_add_name}" |  awk -F"'" '{print $2}'`
+
+    if [[ -z "${existing_cmd}" ]]; then
+      new_cmd="${trap_add_cmd}"
+    else
+      new_cmd="${existing_cmd};${trap_add_cmd}"
+    fi
+
+    # Assign the test
+    trap "${new_cmd}" "${trap_add_name}"
+  done
+}
+
+# Opposite of kube::util::ensure-temp-dir()
+kube::util::cleanup-temp-dir() {
+  rm -rf "${KUBE_TEMP}"
+}
+
 # Create a temp dir that'll be deleted at the end of this bash session.
 #
 # Vars set:
 #   KUBE_TEMP
 kube::util::ensure-temp-dir() {
   if [[ -z ${KUBE_TEMP-} ]]; then
     KUBE_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t kubernetes.XXXXXX)
+    kube::util::trap_add kube::util::cleanup-temp-dir EXIT
   fi
 }
 
@@ -124,60 +155,49 @@ kube::util::wait-for-jobs() {
   return ${fail}
 }
 
-# Takes a binary to run $1 and then copies the results to $2.
-# If the generated and original files are the same after filtering lines
-# that match $3, copy is skipped.
-kube::util::gen-doc() {
-  local cmd="$1"
-  local base_dest="$(kube::util::realpath $2)"
-  local relative_doc_dest="$(echo $3 | sed 's/\/$//')"
-  local dest="${base_dest}/${relative_doc_dest}"
-  local skipprefix="${4:-}"
-
-  # We do this in a tmpdir in case the dest has other non-autogenned files
-  # We don't want to include them in the list of gen'd files
-  local tmpdir="${KUBE_ROOT}/doc_tmp"
-  mkdir -p "${tmpdir}"
-  # generate the new files
-  ${cmd} "${tmpdir}"
+# Run all known doc generators (today gendocs, genman, and genbashcomp for kubectl)
+# $1 is the directory to put those generated documents
+kube::util::gen-docs() {
+  local dest="$1"
+
+  # Find binary
+  gendocs=$(kube::util::find-binary "gendocs")
+  genman=$(kube::util::find-binary "genman")
+  genbashcomp=$(kube::util::find-binary "genbashcomp")
+
+  mkdir -p "${dest}/docs/user-guide/kubectl/"
+  "${gendocs}" "${dest}/docs/user-guide/kubectl/"
+  mkdir -p "${dest}/docs/man/man1/"
+  "${genman}" "${dest}/docs/man/man1/"
+  mkdir -p "${dest}/contrib/completions/bash/"
+  "${genbashcomp}" "${dest}/contrib/completions/bash/"
+
   # create the list of generated files
-  ls "${tmpdir}" | LC_ALL=C sort > "${tmpdir}/.files_generated"
+  pushd "${dest}" > /dev/null
+  touch .generated_docs
+  find -type f | cut -sd / -f 2- | LC_ALL=C sort > .generated_docs
+  popd > /dev/null
 
   while read file; do
-    # Don't add analytics link to generated .md files -- that is done (and
-    # checked) by mungedocs.
-
-    # Remove all old generated files from the destination
-    if [[ -e "${tmpdir}/${file}" ]]; then
-      local original generated
+    # Copy out of KUBE_ROOT if we didn't really change anything
+    if [[ -e "${dest}/${file}" && -e "${KUBE_ROOT}/${file}" ]]; then
       # Filter all munges from original content.
-      original=$(cat "${dest}/${file}" | sed '/^<!-- BEGIN MUNGE:.*/,/^<!-- END MUNGE:.*/d')
-      generated=$(cat "${tmpdir}/${file}")
-      # If this function was asked to filter lines with a prefix, do so.
-      if [[ -n "${skipprefix}" ]]; then
-        original=$(echo "${original}" | grep -v "^${skipprefix}" || :)
-        generated=$(echo "${generated}" | grep -v "^${skipprefix}" || :)
-      fi
+      original=$(cat "${KUBE_ROOT}/${file}" | sed '/^<!-- BEGIN MUNGE:.*/,/^<!-- END MUNGE:.*/d')
+      generated=$(cat "${dest}/${file}")
+
+      # Filter out meaningless lines with timestamps
+      original=$(echo "${original}" | grep -v "Auto generated by spf13/cobra" || :)
+      generated=$(echo "${generated}" | grep -v "Auto generated by spf13/cobra" || :)
+
       # By now, the contents should be normalized and stripped of any
       # auto-managed content.  We also ignore whitespace here because of
       # markdown strictness fixups later in the pipeline.
-      if diff -Bw <(echo "${original}") <(echo "${generated}") > /dev/null; then
+      if diff -Bw >/dev/null <(echo "${original}") <(echo "${generated}"); then
         # actual contents same, overwrite generated with original.
-        mv "${dest}/${file}" "${tmpdir}/${file}"
+        cp "${KUBE_ROOT}/${file}" "${dest}/${file}"
       fi
-    else
-      rm "${dest}/${file}" || true
     fi
-  done <"${dest}/.files_generated"
-
-  # put the new generated file into the destination
-  # the shopt is so that we get .files_generated from the glob.
-  shopt -s dotglob
-  cp -af "${tmpdir}"/* "${dest}"
-  shopt -u dotglob
-
-  #cleanup
-  rm -rf "${tmpdir}"
+  done <"${KUBE_ROOT}/.generated_docs"
 }
 
 # Takes a path $1 to traverse for md files to append the ga-beacon tracking

diff --git a/hack/test-cmd.sh b/hack/test-cmd.sh
@@ -74,8 +74,7 @@ function check-curl-proxy-code()
   echo "For address ${full_address}, got ${status} but wanted ${desired}"
   return 1
 }
-
-trap cleanup EXIT SIGINT
+kube::util::trap_add cleanup EXIT SIGINT
 
 kube::util::ensure-temp-dir
 kube::etcd::start