Skip to content

Commit

Permalink
Rewrite how the munger works
Browse files Browse the repository at this point in the history 
The basic idea is that in the main mungedocs we run the entirefile and
create an annotated set of lines about that file. All mungers then act
on a struct mungeLines instead of on a bytes array. Making use of the
metadata where appropriete. Helper functions exist to make updating a
'macro block' extremely easy.
  • Loading branch information
@eparis
eparis committed Jul 31, 2015
1 parent 4cbca2e commit 8886a99
Show file tree
Hide file tree
Showing 22 changed files with 753 additions and 507 deletions.
22 changes: 22 additions & 0 deletions cmd/mungedocs/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Documentation Mungers

Basically this is like lint/gofmt for md docs.

It basically does the following:
- iterate over all files in the given doc root.
- for each file split it into a slice (mungeLines) of lines (mungeLine)
- a mungeline has metadata about each line typically determined by a 'fast' regex.
- metadata contains things like 'is inside a preformmatted block'
- contains a markdown header
- has a link to another file
- etc..
- if you have a really slow regex with a lot of backtracking you might want to write a fast one to limit how often you run the slow one.
- each munger is then called in turn
- they are given the mungeLines
- they create an entirely new set of mungeLines with their modifications
- the new set is returned
- the new set is then fed into the next munger.
- in the end we might commit the end mungeLines to the file or not (--verify)


[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cmd/mungedocs/README.md?pixel)]()
63 changes: 31 additions & 32 deletions cmd/mungedocs/analytics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,43 +17,42 @@ limitations under the License.
package main

import (
"bytes"
"fmt"
"os"
"regexp"
"strings"
)

var (
beginMungeExp = regexp.QuoteMeta(beginMungeTag("GENERATED_ANALYTICS"))
endMungeExp = regexp.QuoteMeta(endMungeTag("GENERATED_ANALYTICS"))
analyticsExp = regexp.QuoteMeta("[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/") +
"[^?]*" +
regexp.QuoteMeta("?pixel)]()")
const analyticsMungeTag = "GENERATED_ANALYTICS"
const analyticsLinePrefix = "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/"

// Matches the analytics blurb, with or without the munge headers.
analyticsRE = regexp.MustCompile(`[\n]*` + analyticsExp + `[\n]?` +
`|` + `[\n]*` + beginMungeExp + `[^<]*` + endMungeExp)
)

// This adds the analytics link to every .md file.
func checkAnalytics(fileName string, fileBytes []byte) (output []byte, err error) {
fileName = makeRepoRelative(fileName)
desired := fmt.Sprintf(`
func updateAnalytics(fileName string, mlines mungeLines) (mungeLines, error) {
var out mungeLines
fileName, err := makeRepoRelative(fileName, fileName)
if err != nil {
return mlines, err
}

link := fmt.Sprintf(analyticsLinePrefix+"%s?pixel)]()", fileName)
insertLines := getMungeLines(link)
mlines, err = removeMacroBlock(analyticsMungeTag, mlines)
if err != nil {
return mlines, err
}

`+beginMungeTag("GENERATED_ANALYTICS")+`
[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/%s?pixel)]()
`+endMungeTag("GENERATED_ANALYTICS")+`
`, fileName)
if !analyticsRE.MatchString(desired) {
fmt.Printf("%q does not match %q", analyticsRE.String(), desired)
os.Exit(1)
// Remove floating analytics links not surrounded by the munge tags.
for _, mline := range mlines {
if mline.preformatted || mline.header || mline.beginTag || mline.endTag {
out = append(out, mline)
continue
}
if strings.HasPrefix(mline.data, analyticsLinePrefix) {
continue
}
out = append(out, mline)
}
out = appendMacroBlock(out, analyticsMungeTag)
out, err = updateMacroBlock(out, analyticsMungeTag, insertLines)
if err != nil {
return mlines, err
}
//output = replaceNonPreformattedRegexp(fileBytes, analyticsRE, func(in []byte) []byte {
output = analyticsRE.ReplaceAllFunc(fileBytes, func(in []byte) []byte {
return []byte{}
})
output = bytes.TrimRight(output, "\n")
output = append(output, []byte(desired)...)
return output, nil
return out, nil
}
64 changes: 34 additions & 30 deletions cmd/mungedocs/analytics_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,67 +23,71 @@ import (
)

func TestAnalytics(t *testing.T) {
b := beginMungeTag("GENERATED_ANALYTICS")
e := endMungeTag("GENERATED_ANALYTICS")
var cases = []struct {
in string
out string
in string
expected string
}{
{
"aoeu",
"aoeu" + "\n" + "\n" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
"aoeu" + "\n" + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") + "\n"},
e + "\n"},
{
"aoeu" + "\n" + "\n" + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()",
"aoeu" + "\n" + "\n" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") + "\n"},
e + "\n"},
{
"aoeu" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") + "\n",
"aoeu" + "\n" + "\n" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
e + "\n",
"aoeu" + "\n" + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") + "\n"},
e + "\n"},
{
"aoeu" + "\n" + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + "\n" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") + "\n",
"aoeu" + "\n" + "\n" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
e + "\n",
"aoeu" + "\n" + "\n" + "\n" + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") + "\n"},
e + "\n"},
{
"prefix" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") +
e +
"\n" + "suffix",
"prefix" + "\n" + "suffix" + "\n" + "\n" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
"prefix" + "\n" + "suffix" + "\n" + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") + "\n"},
e + "\n"},
{
"aoeu" + "\n" + "\n" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") + "\n",
e + "\n",
"aoeu" + "\n" + "\n" + "\n" +
beginMungeTag("GENERATED_ANALYTICS") + "\n" +
b + "\n" +
"[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" +
endMungeTag("GENERATED_ANALYTICS") + "\n"},
e + "\n"},
}
for _, c := range cases {
out, err := checkAnalytics("path/to/file-name.md", []byte(c.in))
for i, c := range cases {
in := getMungeLines(c.in)
expected := getMungeLines(c.expected)
out, err := updateAnalytics("path/to/file-name.md", in)
assert.NoError(t, err)
if string(out) != c.out {
t.Errorf("Expected \n\n%v\n\n but got \n\n%v\n\n", c.out, string(out))
if !expected.Equal(out) {
t.Errorf("Case %d Expected \n\n%v\n\n but got \n\n%v\n\n", i, expected.String(), out.String())
}
}
}
127 changes: 62 additions & 65 deletions cmd/mungedocs/example_syncer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,17 @@ limitations under the License.
package main

import (
"bytes"
"fmt"
"io/ioutil"
"path"
"regexp"
"strings"
)

const exampleMungeTag = "EXAMPLE"
const exampleToken = "EXAMPLE"

const exampleLineStart = "<!-- BEGIN MUNGE: EXAMPLE"

var exampleMungeTagRE = regexp.MustCompile(beginMungeTag(fmt.Sprintf("%s %s", exampleToken, `(([^ ])*.(yaml|json))`)))

// syncExamples updates all examples in markdown file.
//
Expand All @@ -43,75 +45,70 @@ const exampleMungeTag = "EXAMPLE"
//
// [Download example](../../examples/guestbook/frontend-controller.yaml)
// <!-- END MUNGE: EXAMPLE -->
func syncExamples(filePath string, markdown []byte) ([]byte, error) {
// find the example syncer begin tag
header := beginMungeTag(fmt.Sprintf("%s %s", exampleMungeTag, `(([^ ])*.(yaml|json))`))
exampleLinkRE := regexp.MustCompile(header)
lines := splitLines(markdown)
updatedMarkdown, err := updateExampleMacroBlock(filePath, lines, exampleLinkRE, endMungeTag(exampleMungeTag))
if err != nil {
return updatedMarkdown, err
func syncExamples(filePath string, mlines mungeLines) (mungeLines, error) {
var err error
type exampleTag struct {
token string
linkText string
fileType string
}
exampleTags := []exampleTag{}

// collect all example Tags
for _, mline := range mlines {
if mline.preformatted || !mline.beginTag {
continue
}
line := mline.data
if !strings.HasPrefix(line, exampleLineStart) {
continue
}
match := exampleMungeTagRE.FindStringSubmatch(line)
if len(match) < 4 {
err = fmt.Errorf("Found unparsable EXAMPLE munge line %v", line)
return mlines, err
}
tag := exampleTag{
token: exampleToken + " " + match[1],
linkText: match[1],
fileType: match[3],
}
exampleTags = append(exampleTags, tag)
}
// update all example Tags
for _, tag := range exampleTags {
example, err := exampleContent(filePath, tag.linkText, tag.fileType)
if err != nil {
return mlines, err
}
mlines, err = updateMacroBlock(mlines, tag.token, example)
if err != nil {
return mlines, err
}
}
return updatedMarkdown, nil
return mlines, nil
}

// exampleContent retrieves the content of the file at linkPath
func exampleContent(filePath, linkPath, fileType string) (content string, err error) {
realRoot := path.Join(*rootDir, *repoRoot) + "/"
path := path.Join(realRoot, path.Dir(filePath), linkPath)
dat, err := ioutil.ReadFile(path)
func exampleContent(filePath, linkPath, fileType string) (mungeLines, error) {
repoRel, err := makeRepoRelative(linkPath, filePath)
if err != nil {
return content, err
return nil, err
}
// remove leading and trailing spaces and newlines
trimmedFileContent := strings.TrimSpace(string(dat))
content = fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s)", fileType, trimmedFileContent, linkPath)
return
}

// updateExampleMacroBlock sync the yaml/json example between begin tag and end tag
func updateExampleMacroBlock(filePath string, lines []string, beginMarkExp *regexp.Regexp, endMark string) ([]byte, error) {
var buffer bytes.Buffer
betweenBeginAndEnd := false
for _, line := range lines {
trimmedLine := strings.Trim(line, " \n")
if beginMarkExp.Match([]byte(trimmedLine)) {
if betweenBeginAndEnd {
return nil, fmt.Errorf("found second begin mark while updating macro blocks")
}
betweenBeginAndEnd = true
buffer.WriteString(line)
buffer.WriteString("\n")
match := beginMarkExp.FindStringSubmatch(line)
if len(match) < 4 {
return nil, fmt.Errorf("failed to parse the link in example header")
}
// match[0] is the entire expression; [1] is the link text and [3] is the file type (yaml or json).
linkText := match[1]
fileType := match[3]
example, err := exampleContent(filePath, linkText, fileType)
if err != nil {
return nil, err
}
buffer.WriteString(example)
} else if trimmedLine == endMark {
if !betweenBeginAndEnd {
return nil, fmt.Errorf("found end mark without being mark while updating macro blocks")
}
// Extra newline avoids github markdown bug where comment ends up on same line as last bullet.
buffer.WriteString("\n")
buffer.WriteString(line)
buffer.WriteString("\n")
betweenBeginAndEnd = false
} else {
if !betweenBeginAndEnd {
buffer.WriteString(line)
buffer.WriteString("\n")
}
}
fileRel, err := makeFileRelative(linkPath, filePath)
if err != nil {
return nil, err
}
if betweenBeginAndEnd {
return nil, fmt.Errorf("never found closing end mark while updating macro blocks")

dat, err := ioutil.ReadFile(repoRel)
if err != nil {
return nil, err
}
return buffer.Bytes(), nil

// remove leading and trailing spaces and newlines
trimmedFileContent := strings.TrimSpace(string(dat))
content := fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s)", fileType, trimmedFileContent, fileRel)
out := getMungeLines(content)
return out, nil
}
21 changes: 12 additions & 9 deletions cmd/mungedocs/example_syncer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,24 +35,27 @@ spec:
- containerPort: 80
`
var cases = []struct {
in string
out string
in string
expected string
}{
{"", ""},
{
"<!-- BEGIN MUNGE: EXAMPLE testdata/pod.yaml -->\n<!-- END MUNGE: EXAMPLE -->\n",
"<!-- BEGIN MUNGE: EXAMPLE testdata/pod.yaml -->\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml)\n<!-- END MUNGE: EXAMPLE -->\n",
"<!-- BEGIN MUNGE: EXAMPLE testdata/pod.yaml -->\n<!-- END MUNGE: EXAMPLE testdata/pod.yaml -->\n",
"<!-- BEGIN MUNGE: EXAMPLE testdata/pod.yaml -->\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml)\n<!-- END MUNGE: EXAMPLE testdata/pod.yaml -->\n",
},
{
"<!-- BEGIN MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n<!-- END MUNGE: EXAMPLE -->\n",
"<!-- BEGIN MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml)\n<!-- END MUNGE: EXAMPLE -->\n",
"<!-- BEGIN MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n<!-- END MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n",
"<!-- BEGIN MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml)\n<!-- END MUNGE: EXAMPLE ../mungedocs/testdata/pod.yaml -->\n",
},
}
repoRoot = ""
for _, c := range cases {
actual, err := syncExamples("mungedocs/filename.md", []byte(c.in))
in := getMungeLines(c.in)
expected := getMungeLines(c.expected)
actual, err := syncExamples("filename.md", in)
assert.NoError(t, err)
if c.out != string(actual) {
t.Errorf("Expected example \n'%v' but got \n'%v'", c.out, string(actual))
if !expected.Equal(actual) {
t.Errorf("Expected example \n'%q' but got \n'%q'", expected.String(), actual.String())
}
}
}
Loading

0 comments on commit 8886a99

Please sign in to comment.