Skip to content

Commit

Permalink
Introduce new transaction description format
Browse files Browse the repository at this point in the history 
The new format allows for multiline strings.

Old format:

2024-09-27 "foobar"
Assets:A Assets:B 45 USD

New format:

2024-09-27
| foobar
Assets:A Assets:B 45 USD
  • Loading branch information
Silvio Böhler committed Sep 27, 2024
1 parent 582e928 commit 528ec25
Show file tree
Hide file tree
Showing 9 changed files with 189 additions and 75 deletions.
6 changes: 4 additions & 2 deletions cmd/commands/testdata/infer/target.golden
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
2021-06-18 "foo2"
2021-06-18
| foo2
Assets:Bankaccount Expenses:Foo2 50 USD

2021-06-18 "something"
2021-06-18
| something
Assets:Bankaccount Expenses:Baz 50 USD
7 changes: 6 additions & 1 deletion lib/model/transaction/transaction.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package transaction

import (
"fmt"
"strings"
"time"

"github.com/sboehler/knut/lib/common/compare"
Expand Down Expand Up @@ -63,7 +64,11 @@ func Create(reg *registry.Registry, t *syntax.Transaction) ([]*Transaction, erro
if err != nil {
return nil, err
}
desc := t.Description.Content.Extract()
var ss []string
for _, content := range t.Description {
ss = append(ss, content.Content.Extract())
}
desc := strings.Join(ss, "\n")
postings, err := posting.Create(reg, t.Bookings)
if err != nil {
return nil, err
Expand Down
6 changes: 5 additions & 1 deletion lib/syntax/bayes/bayes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,11 @@ func (m *Model) scoreCandidate(candidate string, tokens set.Set[token]) float64
}

func tokenize(t *syntax.Transaction, b *syntax.Booking, other string) set.Set[token] {
tokens := append(strings.Fields(t.Description.Content.Extract()), b.Commodity.Extract(), b.Quantity.Extract(), other)
var tokens []string
tokens = append(tokens, b.Commodity.Extract(), b.Quantity.Extract(), other)
for _, d := range t.Description {
tokens = append(tokens, strings.Fields(d.Content.Extract())...)
}
result := set.New[token]()
for _, t := range tokens {
result.Add(token(strings.ToLower(t)))
Expand Down
63 changes: 36 additions & 27 deletions lib/syntax/bayes/bayes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,35 +20,44 @@ func TestPrintFile(t *testing.T) {
{
desc: "print transaction",
training: lines(
`2022-03-03 "Hello world"`,
`A B 400 CHF`,
``,
`2022-03-03 "Hello Europe"`,
`A C 400 CHF`,
``,
`2022-03-03 "Hello Asia"`,
`A D 400 CHF`,
``,
"2022-03-03",
"| Hello world",
"A B 400 CHF",
"",
"2022-03-03",
"| Hello Europe",
"A C 400 CHF",
"",
"2022-03-03",
"| Hello Asia",
"A D 400 CHF",
"",
),
target: lines(
`2022-03-03 "hello europe"`,
`A TBD 400 CHF`,
``,
`2022-03-03 "hello world"`,
`A TBD 400 CHF`,
``,
`2022-03-03 "hello asia"`,
`A TBD 400 CHF`,
"2022-03-03",
"| hello europe",
"A TBD 400 CHF",
"",
"2022-03-03",
"| hello world",
"A TBD 400 CHF",
"",
"2022-03-03",
"| hello asia",
"A TBD 400 CHF",
),
want: lines(
`2022-03-03 "hello europe"`,
`A C 400 CHF`,
``,
`2022-03-03 "hello world"`,
`A B 400 CHF`,
``,
`2022-03-03 "hello asia"`,
`A D 400 CHF`,
"2022-03-03",
"| hello europe",
"A C 400 CHF",
"",
"2022-03-03",
"| hello world",
"A B 400 CHF",
"",
"2022-03-03",
"| hello asia",
"A D 400 CHF",
),
},
}
Expand All @@ -74,7 +83,7 @@ func TestPrintFile(t *testing.T) {
err := syntax.FormatFile(&got, target)

if err != nil {
t.Fatalf("pr.Format() returned unexpected error: %v", err)
t.Fatalf("pr.Format() returned unexpected error: %v", err.Error())
}
if diff := cmp.Diff(test.want, got.String()); diff != "" {
t.Fatalf("PrintFile() returned unexpected diff (-want/+got):\n%s\n", diff)
Expand All @@ -95,7 +104,7 @@ func parse(t *testing.T, s string) syntax.File {
}
f, err := p.ParseFile()
if err != nil {
t.Fatalf("p.ParseFile() returned unexpected error: %#v", err)
t.Fatalf("p.ParseFile() returned unexpected error: %v", err)
}
return f
}
4 changes: 3 additions & 1 deletion lib/syntax/directives/directives.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ type Addons struct {
type Transaction struct {
Range
Date Date
Description QuotedString
Description []QuotedString
Bookings []Booking
Addons Addons
}
Expand Down Expand Up @@ -229,6 +229,8 @@ func (e Error) Error() string {
if e.Wrapped != nil {
s.WriteString(e.Wrapped.Error())
s.WriteString("\n")
} else {
s.WriteString(e.Text)
}
if len(e.Path) > 0 {
s.WriteString(e.Path)
Expand Down
48 changes: 40 additions & 8 deletions lib/syntax/parser/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (p *Parser) parseDirective() (directives.Directive, error) {
if _, err := p.readWhitespace1(); err != nil {
return directives.SetRange(&dir, s.Range()), s.Annotate(err)
}
if p.Current() == '"' {
if p.Current() == '"' || p.Current() == '\n' {
if dir.Directive, err = p.parseTransaction(s, date, addons); err != nil {
return directives.SetRange(&dir, s.Range()), s.Annotate(err)
}
Expand Down Expand Up @@ -379,17 +379,49 @@ func (p *Parser) parseQuotedString() (directives.QuotedString, error) {
return directives.SetRange(&qs, s.Range()), nil
}

func (p *Parser) parseTransaction(s scanner.Scope, date directives.Date, addons directives.Addons) (directives.Transaction, error) {
s.UpdateDesc("parsing transaction")
func (p *Parser) parseDescriptionLine() (directives.QuotedString, error) {
s := p.Scope("parsing description line")
var (
trx = directives.Transaction{Date: date, Addons: addons}
qs directives.QuotedString
err error
)
if trx.Description, err = p.parseQuotedString(); err != nil {
return directives.SetRange(&trx, s.Range()), s.Annotate(err)
if _, err := p.ReadString("| "); err != nil {
return directives.SetRange(&qs, s.Range()), s.Annotate(err)
}
if _, err := p.readRestOfWhitespaceLine(); err != nil {
return directives.SetRange(&trx, s.Range()), s.Annotate(err)
if qs.Content, err = p.ReadWhile(func(r rune) bool { return r != '\n' }); err != nil {
return directives.SetRange(&qs, s.Range()), s.Annotate(err)
}
if _, err := p.ReadCharacter('\n'); err != nil {
return directives.SetRange(&qs, s.Range()), s.Annotate(err)
}
return directives.SetRange(&qs, s.Range()), nil
}

func (p *Parser) parseTransaction(s scanner.Scope, date directives.Date, addons directives.Addons) (directives.Transaction, error) {
s.UpdateDesc("parsing transaction")
trx := directives.Transaction{Date: date, Addons: addons}

switch p.Current() {
case '"':
desc, err := p.parseQuotedString()
trx.Description = []directives.QuotedString{desc}
if err != nil {
return directives.SetRange(&trx, s.Range()), s.Annotate(err)
}
if _, err := p.readRestOfWhitespaceLine(); err != nil {
return directives.SetRange(&trx, s.Range()), s.Annotate(err)
}
case '\n':
if _, err := p.Scanner.ReadCharacter('\n'); err != nil {
return directives.SetRange(&trx, s.Range()), s.Annotate(err)
}
for p.Current() == '|' {
line, err := p.parseDescriptionLine()
if err != nil {
return directives.SetRange(&trx, s.Range()), s.Annotate(err)
}
trx.Description = append(trx.Description, line)
}
}
for {
b, err := p.parseBooking()
Expand Down
103 changes: 78 additions & 25 deletions lib/syntax/parser/parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,11 @@ func TestParseFile(t *testing.T) {
Directive: directives.Transaction{
Range: directives.Range{End: 48, Text: s},
Date: directives.Date{Range: Range{End: 10, Text: s}},
Description: directives.QuotedString{
Range: Range{Start: 11, End: 25, Text: s},
Content: Range{Start: 12, End: 24, Text: s},
Description: []directives.QuotedString{
{
Range: Range{Start: 11, End: 25, Text: s},
Content: Range{Start: 12, End: 24, Text: s},
},
},
Bookings: []directives.Booking{
{
Expand Down Expand Up @@ -1035,6 +1037,43 @@ func TestParseQuotedString(t *testing.T) {
}.run(t)
}

func TestParseDescriptionLine(t *testing.T) {
parserTest[directives.QuotedString]{
desc: "p.parseDescriptionLine()",
fn: func(p *Parser) (directives.QuotedString, error) { return p.parseDescriptionLine() },
tests: []testcase[directives.QuotedString]{
{
text: "| foobar\n",
want: func(s string) directives.QuotedString {
return directives.QuotedString{
Range: Range{End: 9, Text: s},
Content: Range{Start: 2, End: 8, Text: s},
}
},
},
{
text: "| foobar",
want: func(s string) directives.QuotedString {
return directives.QuotedString{
Range: Range{End: 8, Text: s},
Content: Range{Start: 2, End: 8, Text: s},
}
},
err: func(s string) error {
return directives.Error{
Message: "while parsing description line",
Range: Range{End: 8, Text: s},
Wrapped: directives.Error{
Range: Range{Start: 8, End: 8, Text: s},
Message: "unexpected end of file, want `\n`",
},
}
},
},
},
}.run(t)
}

func TestParseTransaction(t *testing.T) {
parserTest[directives.Transaction]{
tests: []testcase[directives.Transaction]{
Expand All @@ -1043,9 +1082,11 @@ func TestParseTransaction(t *testing.T) {
want: func(t string) directives.Transaction {
return directives.Transaction{
Range: Range{End: 16, Text: t},
Description: directives.QuotedString{
Range: Range{End: 5, Text: t},
Content: Range{Start: 1, End: 4, Text: t},
Description: []directives.QuotedString{
{
Range: Range{End: 5, Text: t},
Content: Range{Start: 1, End: 4, Text: t},
},
},
Bookings: []directives.Booking{
{
Expand All @@ -1064,9 +1105,11 @@ func TestParseTransaction(t *testing.T) {
want: func(t string) directives.Transaction {
return directives.Transaction{
Range: Range{End: 26, Text: t},
Description: directives.QuotedString{
Range: Range{End: 5, Text: t},
Content: Range{Start: 1, End: 4, Text: t},
Description: []directives.QuotedString{
{
Range: Range{End: 5, Text: t},
Content: Range{Start: 1, End: 4, Text: t},
},
},
Bookings: []directives.Booking{
{
Expand All @@ -1092,9 +1135,11 @@ func TestParseTransaction(t *testing.T) {
want: func(t string) directives.Transaction {
return directives.Transaction{
Range: Range{End: 15, Text: t},
Description: directives.QuotedString{
Range: Range{End: 5, Text: t},
Content: Range{Start: 1, End: 4, Text: t},
Description: []directives.QuotedString{
{
Range: Range{End: 5, Text: t},
Content: Range{Start: 1, End: 4, Text: t},
},
},
Bookings: []directives.Booking{
{
Expand All @@ -1113,9 +1158,11 @@ func TestParseTransaction(t *testing.T) {
want: func(t string) directives.Transaction {
return directives.Transaction{
Range: Range{End: 9, Text: t},
Description: directives.QuotedString{
Range: Range{End: 5, Text: t},
Content: Range{Start: 1, End: 4, Text: t},
Description: []directives.QuotedString{
{
Range: Range{End: 5, Text: t},
Content: Range{Start: 1, End: 4, Text: t},
},
},
Bookings: []directives.Booking{
{
Expand Down Expand Up @@ -1161,9 +1208,11 @@ func TestParseDirective(t *testing.T) {
Directive: directives.Transaction{
Range: Range{End: 45, Text: s},
Date: directives.Date{Range: directives.Range{Start: 18, End: 28, Text: s}},
Description: directives.QuotedString{
Range: Range{Start: 29, End: 34, Text: s},
Content: Range{Start: 30, End: 33, Text: s},
Description: []directives.QuotedString{
{
Range: Range{Start: 29, End: 34, Text: s},
Content: Range{Start: 30, End: 33, Text: s},
},
},
Bookings: []directives.Booking{
{
Expand Down Expand Up @@ -1196,9 +1245,11 @@ func TestParseDirective(t *testing.T) {
Directive: directives.Transaction{
Range: Range{End: 27, Text: s},
Date: directives.Date{Range: directives.Range{End: 10, Text: s}},
Description: directives.QuotedString{
Range: Range{Start: 11, End: 16, Text: s},
Content: Range{Start: 12, End: 15, Text: s},
Description: []directives.QuotedString{
{
Range: Range{Start: 11, End: 16, Text: s},
Content: Range{Start: 12, End: 15, Text: s},
},
},
Bookings: []directives.Booking{
{
Expand Down Expand Up @@ -1243,9 +1294,11 @@ func TestParseDirective(t *testing.T) {
Directive: directives.Transaction{
Range: Range{End: 15, Text: s},
Date: directives.Date{Range: directives.Range{End: 10, Text: s}},
Description: directives.QuotedString{
Range: directives.Range{Start: 11, End: 15, Text: s},
Content: directives.Range{Start: 12, End: 15, Text: s},
Description: []directives.QuotedString{
{
Range: directives.Range{Start: 11, End: 15, Text: s},
Content: directives.Range{Start: 12, End: 15, Text: s},
},
},
},
}
Expand Down Expand Up @@ -1381,7 +1434,7 @@ func TestParseDirective(t *testing.T) {

func TestParseRestOfWhitespaceLine(t *testing.T) {
parserTest[Range]{
desc: "p.parseQuotedString()",
desc: "p.readRestOfWhitespaceLine()",
fn: func(p *Parser) (Range, error) { return p.readRestOfWhitespaceLine() },
tests: []testcase[Range]{
{
Expand Down
Loading

0 comments on commit 528ec25

Please sign in to comment.