Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Perf/lexer performance improvement #372

Merged
merged 3 commits into from
Oct 19, 2024
Merged

Perf/lexer performance improvement #372

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d535a57
perf: use buffer pool
ysugimoto Oct 19, 2024
8abec01
improve underlying lexer performance
ysugimoto Oct 19, 2024
7118cb4
fix nolint
ysugimoto Oct 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
improve underlying lexer performance
  • Loading branch information
@ysugimoto
ysugimoto committed Oct 19, 2024
commit 8abec0190168b7749c0e7e6a390056bc1b390318
107 changes: 3 additions & 104 deletions lexer/lexer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ func New(r io.Reader, opts ...OptionFunc) *Lexer {
r: bufio.NewReader(r),
line: 1,
buffer: new(bytes.Buffer),
stack: make([]string, 0, 512),
peeks: make([]token.Token, 0, 8),
file: o.Filename,
customs: o.Customs,
}
Expand Down Expand Up @@ -416,111 +418,8 @@ func (l *Lexer) skipWhitespace() {
}
}

func (l *Lexer) readString() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck
defer pool.Put(buf)
buf.Reset()

l.readChar()
for {
if l.char == '"' || l.char == 0x00 {
break
}
buf.WriteRune(l.char)
l.readChar()
}

return buf.String()
}

func (l *Lexer) readBracketString() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck
defer pool.Put(buf)
buf.Reset()

l.readChar()
for {
if l.char == 0x00 {
break
}
if l.char == '"' {
if l.peekChar() == '}' {
l.readChar()
break
}
}
buf.WriteRune(l.char)
l.readChar()
}

return buf.String()
}

func (l *Lexer) readNumber() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck
defer pool.Put(buf)
buf.Reset()

for isDigit(l.char) {
buf.WriteRune(l.char)
l.readChar()
}
return buf.String()
}

func (l *Lexer) readEOL() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck
defer pool.Put(buf)
buf.Reset()

for {
buf.WriteRune(l.char)
if l.peekChar() == 0x00 || l.peekChar() == '\n' {
break
}
l.readChar()
}
return buf.String()
}

func (l *Lexer) readMultiComment() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck
defer pool.Put(buf)
buf.Reset()

for {
if l.char == 0x00 {
break
}
if l.char == '*' && l.peekChar() == '/' {
buf.WriteRune(l.char)
l.readChar()
buf.WriteRune(l.char)
break
}
buf.WriteRune(l.char)
l.readChar()
}

return buf.String()
}

func (l *Lexer) readIdentifier() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck
defer pool.Put(buf)
buf.Reset()

for l.isLetter(l.char) {
buf.WriteRune(l.char)
l.readChar()
}
return buf.String()
}

func (l *Lexer) isLetter(r rune) bool {
// Letter allows "-", "." and ":" character to parse ident http header name like `req.http.X-Forwarded-For`
// but in some name definitions (e.g. acl name, backend name), parser must check "-" IS NOT included.
// return 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || r == '_' || r == '-' || r == '.' || r == ':'
// Letter allows [a-zA-Z_] character to parse ident of http header name like `req`, `http`, `X-Forwarded-For`.
return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r == '_'
}

Expand Down
12 changes: 0 additions & 12 deletions lexer/pool.go
View file
Open in desktop

This file was deleted.

113 changes: 113 additions & 0 deletions lexer/reader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package lexer

import (
"bytes"
"sync"
)

var pool = sync.Pool{
New: func() any {
return new(bytes.Buffer)
},
}

func (l *Lexer) readString() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck

Check failure on line 15 in lexer/reader.go

View workflow job for this annotation

GitHub Actions / build 

Error return value is not checked (errcheck)
defer pool.Put(buf)
buf.Reset()

l.readChar()
for {
if l.char == '"' || l.char == 0x00 {
break
}
buf.WriteRune(l.char)
l.readChar()
}

return buf.String()
}

func (l *Lexer) readBracketString() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck

Check failure on line 32 in lexer/reader.go

View workflow job for this annotation

GitHub Actions / build 

Error return value is not checked (errcheck)
defer pool.Put(buf)
buf.Reset()

l.readChar()
for {
if l.char == 0x00 {
break
}
if l.char == '"' {
if l.peekChar() == '}' {
l.readChar()
break
}
}
buf.WriteRune(l.char)
l.readChar()
}

return buf.String()
}

func (l *Lexer) readNumber() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck

Check failure on line 55 in lexer/reader.go

View workflow job for this annotation

GitHub Actions / build 

Error return value is not checked (errcheck)
defer pool.Put(buf)
buf.Reset()

for isDigit(l.char) {
buf.WriteRune(l.char)
l.readChar()
}
return buf.String()
}

func (l *Lexer) readEOL() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck
defer pool.Put(buf)
buf.Reset()

for {
buf.WriteRune(l.char)
if l.peekChar() == 0x00 || l.peekChar() == '\n' {
break
}
l.readChar()
}
return buf.String()
}

func (l *Lexer) readMultiComment() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck
defer pool.Put(buf)
buf.Reset()

for {
if l.char == 0x00 {
break
}
if l.char == '*' && l.peekChar() == '/' {
buf.WriteRune(l.char)
l.readChar()
buf.WriteRune(l.char)
break
}
buf.WriteRune(l.char)
l.readChar()
}

return buf.String()
}

func (l *Lexer) readIdentifier() string {
buf := pool.Get().(*bytes.Buffer) // notlint:errcheck
defer pool.Put(buf)
buf.Reset()

for l.isLetter(l.char) {
buf.WriteRune(l.char)
l.readChar()
}
return buf.String()
}
Loading