Skip to content

Commit

Permalink
Support for in-place modification of docs
Browse files Browse the repository at this point in the history
Adds cmd/mungedocs which is framework for processing
all files under docs/ and either verifying that no changes needed or
making in-place changes.

Did not reuse kube::util::gen-docs because that seemed to be
centered around handling added files, and this pass does not
add files.

Planned uses:
 - table of contents automatic updating
 - linkification
 - internal link checker
 - link-path-relativizer or absolutizer
 - example file syncer
 - header inserter.

Just table-of-contents updating in this PR.

Added Table of Contents to docs/networking.md.

Demonstrates use of new TOC generator presubmit.
Other docs will be added in future PRs.
Additional development will be needed to handle some
of the more complex cases.
  • Loading branch information
erictune committed Jul 8, 2015
1 parent 1fc162d commit c51e35e
Show file tree
Hide file tree
Showing 7 changed files with 385 additions and 7 deletions.
121 changes: 121 additions & 0 deletions cmd/mungedocs/mungedocs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"

flag "github.com/spf13/pflag"
)

var (
verify = flag.Bool("verify", false, "Exit with status 1 if files would have needed changes but do not change.")
rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.")

ErrChangesNeeded = errors.New("mungedocs: changes required")
)

func visitAndVerify(path string, i os.FileInfo, e error) error {
return visitAndChangeOrVerify(path, i, e, false)
}

func visitAndChange(path string, i os.FileInfo, e error) error {
return visitAndChangeOrVerify(path, i, e, true)
}

// Either change a file or verify that it needs no changes (according to modify argument)
func visitAndChangeOrVerify(path string, i os.FileInfo, e error, modify bool) error {
if !strings.HasSuffix(path, ".md") {
return nil
}
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()

before, err := ioutil.ReadAll(file)
if err != nil {
return err
}

after, err := updateTOC(before)
if err != nil {
return err
}
if modify {
// Write out new file with any changes.
if !bytes.Equal(after, before) {
file.Close()
ioutil.WriteFile(path, after, 0644)
}
} else {
// Just verify that there are no changes.
if !bytes.Equal(after, before) {
return ErrChangesNeeded
}
}

// TODO(erictune): more types of passes, such as:
// Linkify terms
// Verify links point to files.

return nil
}

func main() {
flag.Parse()

if *rootDir == "" {
fmt.Fprintf(os.Stderr, "usage: %s [--verify] --root-dir <docs root>\n", flag.Arg(0))
os.Exit(1)
}

// For each markdown file under source docs root, process the doc.
// If any error occurs, will exit with failure.
// If verify is true, then status is 0 for no changes needed, 1 for changes needed
// and >1 for an error during processing.
// If verify is false, then status is 0 if changes successfully made or no changes needed,
// 1 if changes were needed but require human intervention, and >1 for an unexpected
// error during processing.
var err error
if *verify {
err = filepath.Walk(*rootDir, visitAndVerify)
} else {
err = filepath.Walk(*rootDir, visitAndChange)
}
if err != nil {
if err == ErrChangesNeeded {
if *verify {
fmt.Fprintf(os.Stderr,
"Some changes needed but not made due to --verify=true\n")
} else {
fmt.Fprintf(os.Stderr,
"Some changes needed but human intervention is required\n")
}
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "filepath.Walk() returned %v\n", err)
os.Exit(2)
}
}
115 changes: 115 additions & 0 deletions cmd/mungedocs/toc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"bufio"
"bytes"
"fmt"
"strings"
)

// inserts/updates a table of contents in markdown file.
//
// First, builds a ToC.
// Then, finds <!-- BEGIN GENERATED TOC --> and <!-- END GENERATED TOC -->, and replaces anything between those with
// the ToC, thereby updating any previously inserted ToC.
//
// TODO(erictune): put this in own package with tests
func updateTOC(markdown []byte) ([]byte, error) {
toc, err := buildTOC(markdown)
if err != nil {
return nil, err
}
updatedMarkdown, err := updateMacroBlock(markdown, "<!-- BEGIN GENERATED TOC -->", "<!-- END GENERATED TOC -->", string(toc))
if err != nil {
return nil, err
}
return updatedMarkdown, nil
}

// Replaces the text between matching "beginMark" and "endMark" within "document" with "insertThis".
//
// Delimiters should occupy own line.
// Returns copy of document with modifications.
func updateMacroBlock(document []byte, beginMark, endMark, insertThis string) ([]byte, error) {
var buffer bytes.Buffer
lines := strings.Split(string(document), "\n")
// Skip trailing empty string from Split-ing
if len(lines) > 0 && lines[len(lines)-1] == "" {
lines = lines[:len(lines)-1]
}
betweenBeginAndEnd := false
for _, line := range lines {
trimmedLine := strings.Trim(line, " \n")
if trimmedLine == beginMark {
if betweenBeginAndEnd {
return nil, fmt.Errorf("found second begin mark while updating macro blocks")
}
betweenBeginAndEnd = true
buffer.WriteString(line)
buffer.WriteString("\n")
} else if trimmedLine == endMark {
if !betweenBeginAndEnd {
return nil, fmt.Errorf("found end mark without being mark while updating macro blocks")
}
buffer.WriteString(insertThis)
// Extra newline avoids github markdown bug where comment ends up on same line as last bullet.
buffer.WriteString("\n")
buffer.WriteString(line)
buffer.WriteString("\n")
betweenBeginAndEnd = false
} else {
if !betweenBeginAndEnd {
buffer.WriteString(line)
buffer.WriteString("\n")
}
}
}
if betweenBeginAndEnd {
return nil, fmt.Errorf("never found closing end mark while updating macro blocks")
}
return buffer.Bytes(), nil
}

// builds table of contents for markdown file
//
// First scans for all section headers (lines that begin with "#" but not within code quotes)
// and builds a table of contents from those. Assumes bookmarks for those will be
// like #each-word-in-heading-in-lowercases-with-dashes-instead-of-spaces.
// builds the ToC.
func buildTOC(markdown []byte) ([]byte, error) {
var buffer bytes.Buffer
scanner := bufio.NewScanner(bytes.NewReader(markdown))
for scanner.Scan() {
line := scanner.Text()
noSharps := strings.TrimLeft(line, "#")
numSharps := len(line) - len(noSharps)
heading := strings.Trim(noSharps, " \n")
if numSharps > 0 {
indent := strings.Repeat(" ", numSharps-1)
bookmark := strings.Replace(strings.ToLower(heading), " ", "-", -1)
tocLine := fmt.Sprintf("%s- [%s](#%s)\n", indent, heading, bookmark)
buffer.WriteString(tocLine)
}
}
if err := scanner.Err(); err != nil {
return []byte{}, err
}

return buffer.Bytes(), nil
}
101 changes: 101 additions & 0 deletions cmd/mungedocs/toc_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"testing"

"github.com/stretchr/testify/assert"
)

func Test_updateMacroBlock(t *testing.T) {
var cases = []struct {
in string
out string
}{
{"", ""},
{"Lorem ipsum\ndolor sit amet\n",
"Lorem ipsum\ndolor sit amet\n"},
{"Lorem ipsum \n BEGIN\ndolor\nEND\nsit amet\n",
"Lorem ipsum \n BEGIN\nfoo\n\nEND\nsit amet\n"},
}
for _, c := range cases {
actual, err := updateMacroBlock([]byte(c.in), "BEGIN", "END", "foo\n")
assert.NoError(t, err)
if c.out != string(actual) {
t.Errorf("Expected '%v' but got '%v'", c.out, string(actual))
}
}
}

func Test_updateMacroBlock_errors(t *testing.T) {
var cases = []struct {
in string
}{
{"BEGIN\n"},
{"blah\nBEGIN\nblah"},
{"END\n"},
{"blah\nEND\nblah\n"},
{"END\nBEGIN"},
{"BEGIN\nEND\nEND"},
{"BEGIN\nBEGIN\nEND"},
{"BEGIN\nBEGIN\nEND\nEND"},
}
for _, c := range cases {
_, err := updateMacroBlock([]byte(c.in), "BEGIN", "END", "foo")
assert.Error(t, err)
}
}

func Test_buildTOC(t *testing.T) {
var cases = []struct {
in string
out string
}{
{"", ""},
{"Lorem ipsum\ndolor sit amet\n", ""},
{"# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n",
"- [Title](#title)\n - [Section Heading](#section-heading)\n"},
}
for _, c := range cases {
actual, err := buildTOC([]byte(c.in))
assert.NoError(t, err)
if c.out != string(actual) {
t.Errorf("Expected TOC '%v' but got '%v'", c.out, string(actual))
}
}
}

func Test_updateTOC(t *testing.T) {
var cases = []struct {
in string
out string
}{
{"", ""},
{"Lorem ipsum\ndolor sit amet\n",
"Lorem ipsum\ndolor sit amet\n"},
{"# Title\nLorem ipsum \n**table of contents**\n<!-- BEGIN GENERATED TOC -->\nold cruft\n<!-- END GENERATED TOC -->\n## Section Heading\ndolor sit amet\n",
"# Title\nLorem ipsum \n**table of contents**\n<!-- BEGIN GENERATED TOC -->\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n<!-- END GENERATED TOC -->\n## Section Heading\ndolor sit amet\n"},
}
for _, c := range cases {
actual, err := updateTOC([]byte(c.in))
assert.NoError(t, err)
if c.out != string(actual) {
t.Errorf("Expected TOC '%v' but got '%v'", c.out, string(actual))
}
}
}
16 changes: 16 additions & 0 deletions docs/networking.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,20 @@
# Networking in Kubernetes
**Table of Contents**
<!-- BEGIN GENERATED TOC -->
- [Networking in Kubernetes](#networking-in-kubernetes)
- [Summary](#summary)
- [Docker model](#docker-model)
- [Kubernetes model](#kubernetes-model)
- [How to achieve this](#how-to-achieve-this)
- [Google Compute Engine (GCE)](#google-compute-engine-(gce))
- [L2 networks and linux bridging](#l2-networks-and-linux-bridging)
- [Flannel](#flannel)
- [OpenVSwitch](#openvswitch)
- [Weave](#weave)
- [Calico](#calico)
- [Other reading](#other-reading)

<!-- END GENERATED TOC -->

## Summary

Expand Down
1 change: 1 addition & 0 deletions hack/lib/golang.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ kube::golang::test_targets() {
cmd/integration
cmd/gendocs
cmd/genman
cmd/mungedocs
cmd/genbashcomp
cmd/genconversion
cmd/gendeepcopy
Expand Down
Loading

0 comments on commit c51e35e

Please sign in to comment.