forked from kubernetes/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support for in-place modification of docs
Adds cmd/mungedocs which is framework for processing all files under docs/ and either verifying that no changes needed or making in-place changes. Did not reuse kube::util::gen-docs because that seemed to be centered around handling added files, and this pass does not add files. Planned uses: - table of contents automatic updating - linkification - internal link checker - link-path-relativizer or absolutizer - example file syncer - header inserter. Just table-of-contents updating in this PR. Added Table of Contents to docs/networking.md. Demonstrates use of new TOC generator presubmit. Other docs will be added in future PRs. Additional development will be needed to handle some of the more complex cases.
- Loading branch information
Showing
7 changed files
with
385 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,121 @@ | ||
/* | ||
Copyright 2015 The Kubernetes Authors All rights reserved. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package main | ||
|
||
import ( | ||
"bytes" | ||
"errors" | ||
"fmt" | ||
"io/ioutil" | ||
"os" | ||
"path/filepath" | ||
"strings" | ||
|
||
flag "github.com/spf13/pflag" | ||
) | ||
|
||
var ( | ||
verify = flag.Bool("verify", false, "Exit with status 1 if files would have needed changes but do not change.") | ||
rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.") | ||
|
||
ErrChangesNeeded = errors.New("mungedocs: changes required") | ||
) | ||
|
||
func visitAndVerify(path string, i os.FileInfo, e error) error { | ||
return visitAndChangeOrVerify(path, i, e, false) | ||
} | ||
|
||
func visitAndChange(path string, i os.FileInfo, e error) error { | ||
return visitAndChangeOrVerify(path, i, e, true) | ||
} | ||
|
||
// Either change a file or verify that it needs no changes (according to modify argument) | ||
func visitAndChangeOrVerify(path string, i os.FileInfo, e error, modify bool) error { | ||
if !strings.HasSuffix(path, ".md") { | ||
return nil | ||
} | ||
file, err := os.Open(path) | ||
if err != nil { | ||
return err | ||
} | ||
defer file.Close() | ||
|
||
before, err := ioutil.ReadAll(file) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
after, err := updateTOC(before) | ||
if err != nil { | ||
return err | ||
} | ||
if modify { | ||
// Write out new file with any changes. | ||
if !bytes.Equal(after, before) { | ||
file.Close() | ||
ioutil.WriteFile(path, after, 0644) | ||
} | ||
} else { | ||
// Just verify that there are no changes. | ||
if !bytes.Equal(after, before) { | ||
return ErrChangesNeeded | ||
} | ||
} | ||
|
||
// TODO(erictune): more types of passes, such as: | ||
// Linkify terms | ||
// Verify links point to files. | ||
|
||
return nil | ||
} | ||
|
||
func main() { | ||
flag.Parse() | ||
|
||
if *rootDir == "" { | ||
fmt.Fprintf(os.Stderr, "usage: %s [--verify] --root-dir <docs root>\n", flag.Arg(0)) | ||
os.Exit(1) | ||
} | ||
|
||
// For each markdown file under source docs root, process the doc. | ||
// If any error occurs, will exit with failure. | ||
// If verify is true, then status is 0 for no changes needed, 1 for changes needed | ||
// and >1 for an error during processing. | ||
// If verify is false, then status is 0 if changes successfully made or no changes needed, | ||
// 1 if changes were needed but require human intervention, and >1 for an unexpected | ||
// error during processing. | ||
var err error | ||
if *verify { | ||
err = filepath.Walk(*rootDir, visitAndVerify) | ||
} else { | ||
err = filepath.Walk(*rootDir, visitAndChange) | ||
} | ||
if err != nil { | ||
if err == ErrChangesNeeded { | ||
if *verify { | ||
fmt.Fprintf(os.Stderr, | ||
"Some changes needed but not made due to --verify=true\n") | ||
} else { | ||
fmt.Fprintf(os.Stderr, | ||
"Some changes needed but human intervention is required\n") | ||
} | ||
os.Exit(1) | ||
} | ||
fmt.Fprintf(os.Stderr, "filepath.Walk() returned %v\n", err) | ||
os.Exit(2) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
/* | ||
Copyright 2015 The Kubernetes Authors All rights reserved. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package main | ||
|
||
import ( | ||
"bufio" | ||
"bytes" | ||
"fmt" | ||
"strings" | ||
) | ||
|
||
// inserts/updates a table of contents in markdown file. | ||
// | ||
// First, builds a ToC. | ||
// Then, finds <!-- BEGIN GENERATED TOC --> and <!-- END GENERATED TOC -->, and replaces anything between those with | ||
// the ToC, thereby updating any previously inserted ToC. | ||
// | ||
// TODO(erictune): put this in own package with tests | ||
func updateTOC(markdown []byte) ([]byte, error) { | ||
toc, err := buildTOC(markdown) | ||
if err != nil { | ||
return nil, err | ||
} | ||
updatedMarkdown, err := updateMacroBlock(markdown, "<!-- BEGIN GENERATED TOC -->", "<!-- END GENERATED TOC -->", string(toc)) | ||
if err != nil { | ||
return nil, err | ||
} | ||
return updatedMarkdown, nil | ||
} | ||
|
||
// Replaces the text between matching "beginMark" and "endMark" within "document" with "insertThis". | ||
// | ||
// Delimiters should occupy own line. | ||
// Returns copy of document with modifications. | ||
func updateMacroBlock(document []byte, beginMark, endMark, insertThis string) ([]byte, error) { | ||
var buffer bytes.Buffer | ||
lines := strings.Split(string(document), "\n") | ||
// Skip trailing empty string from Split-ing | ||
if len(lines) > 0 && lines[len(lines)-1] == "" { | ||
lines = lines[:len(lines)-1] | ||
} | ||
betweenBeginAndEnd := false | ||
for _, line := range lines { | ||
trimmedLine := strings.Trim(line, " \n") | ||
if trimmedLine == beginMark { | ||
if betweenBeginAndEnd { | ||
return nil, fmt.Errorf("found second begin mark while updating macro blocks") | ||
} | ||
betweenBeginAndEnd = true | ||
buffer.WriteString(line) | ||
buffer.WriteString("\n") | ||
} else if trimmedLine == endMark { | ||
if !betweenBeginAndEnd { | ||
return nil, fmt.Errorf("found end mark without being mark while updating macro blocks") | ||
} | ||
buffer.WriteString(insertThis) | ||
// Extra newline avoids github markdown bug where comment ends up on same line as last bullet. | ||
buffer.WriteString("\n") | ||
buffer.WriteString(line) | ||
buffer.WriteString("\n") | ||
betweenBeginAndEnd = false | ||
} else { | ||
if !betweenBeginAndEnd { | ||
buffer.WriteString(line) | ||
buffer.WriteString("\n") | ||
} | ||
} | ||
} | ||
if betweenBeginAndEnd { | ||
return nil, fmt.Errorf("never found closing end mark while updating macro blocks") | ||
} | ||
return buffer.Bytes(), nil | ||
} | ||
|
||
// builds table of contents for markdown file | ||
// | ||
// First scans for all section headers (lines that begin with "#" but not within code quotes) | ||
// and builds a table of contents from those. Assumes bookmarks for those will be | ||
// like #each-word-in-heading-in-lowercases-with-dashes-instead-of-spaces. | ||
// builds the ToC. | ||
func buildTOC(markdown []byte) ([]byte, error) { | ||
var buffer bytes.Buffer | ||
scanner := bufio.NewScanner(bytes.NewReader(markdown)) | ||
for scanner.Scan() { | ||
line := scanner.Text() | ||
noSharps := strings.TrimLeft(line, "#") | ||
numSharps := len(line) - len(noSharps) | ||
heading := strings.Trim(noSharps, " \n") | ||
if numSharps > 0 { | ||
indent := strings.Repeat(" ", numSharps-1) | ||
bookmark := strings.Replace(strings.ToLower(heading), " ", "-", -1) | ||
tocLine := fmt.Sprintf("%s- [%s](#%s)\n", indent, heading, bookmark) | ||
buffer.WriteString(tocLine) | ||
} | ||
} | ||
if err := scanner.Err(); err != nil { | ||
return []byte{}, err | ||
} | ||
|
||
return buffer.Bytes(), nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,101 @@ | ||
/* | ||
Copyright 2015 The Kubernetes Authors All rights reserved. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package main | ||
|
||
import ( | ||
"testing" | ||
|
||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
func Test_updateMacroBlock(t *testing.T) { | ||
var cases = []struct { | ||
in string | ||
out string | ||
}{ | ||
{"", ""}, | ||
{"Lorem ipsum\ndolor sit amet\n", | ||
"Lorem ipsum\ndolor sit amet\n"}, | ||
{"Lorem ipsum \n BEGIN\ndolor\nEND\nsit amet\n", | ||
"Lorem ipsum \n BEGIN\nfoo\n\nEND\nsit amet\n"}, | ||
} | ||
for _, c := range cases { | ||
actual, err := updateMacroBlock([]byte(c.in), "BEGIN", "END", "foo\n") | ||
assert.NoError(t, err) | ||
if c.out != string(actual) { | ||
t.Errorf("Expected '%v' but got '%v'", c.out, string(actual)) | ||
} | ||
} | ||
} | ||
|
||
func Test_updateMacroBlock_errors(t *testing.T) { | ||
var cases = []struct { | ||
in string | ||
}{ | ||
{"BEGIN\n"}, | ||
{"blah\nBEGIN\nblah"}, | ||
{"END\n"}, | ||
{"blah\nEND\nblah\n"}, | ||
{"END\nBEGIN"}, | ||
{"BEGIN\nEND\nEND"}, | ||
{"BEGIN\nBEGIN\nEND"}, | ||
{"BEGIN\nBEGIN\nEND\nEND"}, | ||
} | ||
for _, c := range cases { | ||
_, err := updateMacroBlock([]byte(c.in), "BEGIN", "END", "foo") | ||
assert.Error(t, err) | ||
} | ||
} | ||
|
||
func Test_buildTOC(t *testing.T) { | ||
var cases = []struct { | ||
in string | ||
out string | ||
}{ | ||
{"", ""}, | ||
{"Lorem ipsum\ndolor sit amet\n", ""}, | ||
{"# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n", | ||
"- [Title](#title)\n - [Section Heading](#section-heading)\n"}, | ||
} | ||
for _, c := range cases { | ||
actual, err := buildTOC([]byte(c.in)) | ||
assert.NoError(t, err) | ||
if c.out != string(actual) { | ||
t.Errorf("Expected TOC '%v' but got '%v'", c.out, string(actual)) | ||
} | ||
} | ||
} | ||
|
||
func Test_updateTOC(t *testing.T) { | ||
var cases = []struct { | ||
in string | ||
out string | ||
}{ | ||
{"", ""}, | ||
{"Lorem ipsum\ndolor sit amet\n", | ||
"Lorem ipsum\ndolor sit amet\n"}, | ||
{"# Title\nLorem ipsum \n**table of contents**\n<!-- BEGIN GENERATED TOC -->\nold cruft\n<!-- END GENERATED TOC -->\n## Section Heading\ndolor sit amet\n", | ||
"# Title\nLorem ipsum \n**table of contents**\n<!-- BEGIN GENERATED TOC -->\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n<!-- END GENERATED TOC -->\n## Section Heading\ndolor sit amet\n"}, | ||
} | ||
for _, c := range cases { | ||
actual, err := updateTOC([]byte(c.in)) | ||
assert.NoError(t, err) | ||
if c.out != string(actual) { | ||
t.Errorf("Expected TOC '%v' but got '%v'", c.out, string(actual)) | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.