diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ef7fb71..cdb47eac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,12 @@ # Changelog +## [v0.12.4](https://github.com/itchyny/gojq/compare/v0.12.3..v0.12.4) (2021-06-01) +* fix numeric conversion of large floating-point numbers in modulo operator +* implement a compiler option for adding custom iterator functions +* implement `gojq.NewIter` function for creating a new iterator from values +* implement `$ARGS.named` for listing command line variables +* remove `debug` and `stderr` functions from the library +* stop printing newlines on `stderr` function for jq compatibility + ## [v0.12.3](https://github.com/itchyny/gojq/compare/v0.12.2..v0.12.3) (2021-04-01) * fix array slicing with infinities and large numbers (`[0][-infinite:infinite], [0][:1e20]`) * fix multiplying strings and modulo by infinities on MIPS 64 architecture diff --git a/Makefile b/Makefile index a1ec11ef..321924d4 100644 --- a/Makefile +++ b/Makefile @@ -74,12 +74,12 @@ test: build go test -v -race ./... .PHONY: lint -lint: $(GOBIN)/golint +lint: $(GOBIN)/staticcheck go vet ./... - golint -set_exit_status ./... + staticcheck -tags debug ./... -$(GOBIN)/golint: - cd && go get golang.org/x/lint/golint +$(GOBIN)/staticcheck: + cd && go get honnef.co/go/tools/cmd/staticcheck .PHONY: maligned maligned: $(GOBIN)/maligned @@ -98,8 +98,8 @@ clean: go clean .PHONY: update +update: export GOPROXY=direct update: - export GOPROXY=direct rm -f go.sum && go get -u -d ./... && go get github.com/mattn/go-runewidth@v0.0.9 && go mod tidy sed -i.bak '/require (/,/)/d' go.dev.mod && rm -f go.dev.{sum,mod.bak} go get -u -d -modfile=go.dev.mod github.com/itchyny/{astgen,timefmt}-go && go generate diff --git a/README.md b/README.md index 56676735..918c38c5 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,9 @@ func main() { - using [`query.Run`](https://pkg.go.dev/github.com/itchyny/gojq#Query.Run) or [`query.RunWithContext`](https://pkg.go.dev/github.com/itchyny/gojq#Query.RunWithContext) - or alternatively, compile the query using [`gojq.Compile`](https://pkg.go.dev/github.com/itchyny/gojq#Compile) and then [`code.Run`](https://pkg.go.dev/github.com/itchyny/gojq#Code.Run) or [`code.RunWithContext`](https://pkg.go.dev/github.com/itchyny/gojq#Code.RunWithContext). You can reuse the `*Code` against multiple inputs to avoid compilation of the same query. - In either case, you cannot use custom type values as the query input. The type should be `[]interface{}` for an array and `map[string]interface{}` for a map (just like decoded to an `interface{}` using the [encoding/json](https://golang.org/pkg/encoding/json/) package). You can't use `[]int` or `map[string]string`, for example. If you want to query your custom struct, marshal to JSON, unmarshal to `interface{}` and use it as the query input. -- Thirdly, iterate through the results using [`iter.Next() (interface{}, bool)`](https://pkg.go.dev/github.com/itchyny/gojq#Iter). The iterator can emit an error so make sure to handle it. Termination is notified by the second returned value of `Next()`. The reason why the return type is not `(interface{}, error)` is that the iterator can emit multiple errors and you can continue after an error. +- Thirdly, iterate through the results using [`iter.Next() (interface{}, bool)`](https://pkg.go.dev/github.com/itchyny/gojq#Iter). The iterator can emit an error so make sure to handle it. The method returns `true` with results, and `false` when the iterator terminates. + - The return type is not `(interface{}, error)` because iterators can emit multiple errors and you can continue after an error. It is difficult for the iterator to tell the termination in this situation. + - Note that the result iterator may emit infinite number of values; `repeat(0)` and `range(infinite)`. It may stuck with no output value; `def f: f; f`. Use `RunWithContext` when you want to limit the execution time. [`gojq.Compile`](https://pkg.go.dev/github.com/itchyny/gojq#Compile) allows to configure the following compiler options. @@ -130,6 +132,7 @@ func main() { - [`gojq.WithEnvironLoader`](https://pkg.go.dev/github.com/itchyny/gojq#WithEnvironLoader) allows to configure the environment variables referenced by `env` and `$ENV`. By default, OS environment variables are not accessible due to security reasons. You can use `gojq.WithEnvironLoader(os.Environ)` if you want. - [`gojq.WithVariables`](https://pkg.go.dev/github.com/itchyny/gojq#WithVariables) allows to configure the variables which can be used in the query. Pass the values of the variables to [`code.Run`](https://pkg.go.dev/github.com/itchyny/gojq#Code.Run) in the same order. - [`gojq.WithFunction`](https://pkg.go.dev/github.com/itchyny/gojq#WithFunction) allows to add a custom internal function. An internal function can return a single value (which can be an error) each invocation. To add a jq function (which may include a comma operator to emit multiple values, `empty` function, accept a filter for its argument, or call another built-in function), use `LoadInitModules` of the module loader. +- [`gojq.WithIterFunction`](https://pkg.go.dev/github.com/itchyny/gojq#WithIterFunction) allows to add a custom iterator function. An iterator function returns an iterator to emit multiple values. You cannot define both iterator and non-iterator functions of the same name (with possibly different arities). You can use [`gojq.NewIter`](https://pkg.go.dev/github.com/itchyny/gojq#NewIter) to convert values or an error to a [`gojq.Iter`](https://pkg.go.dev/github.com/itchyny/gojq#Iter). - [`gojq.WithInputIter`](https://pkg.go.dev/github.com/itchyny/gojq#WithInputIter) allows to use `input` and `inputs` functions. By default, these functions are disabled. ## Bug Tracker diff --git a/cli/cli.go b/cli/cli.go index 33b0b674..d09e3cf0 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -18,7 +18,7 @@ import ( const name = "gojq" -const version = "0.12.3" +const version = "0.12.4" var revision = "HEAD" @@ -177,6 +177,12 @@ Synopsis: cli.argnames = append(cli.argnames, "$"+k) cli.argvalues = append(cli.argvalues, string(val)) } + named := make(map[string]interface{}, len(cli.argnames)) + for i, name := range cli.argnames { + named[name[1:]] = cli.argvalues[i] + } + cli.argnames = append(cli.argnames, "$ARGS") + cli.argvalues = append(cli.argvalues, map[string]interface{}{"named": named}) var arg, fname string if opts.FromFile != "" { src, err := ioutil.ReadFile(opts.FromFile) @@ -217,6 +223,8 @@ Synopsis: gojq.WithModuleLoader(gojq.NewModuleLoader(modulePaths)), gojq.WithEnvironLoader(os.Environ), gojq.WithVariables(cli.argnames), + gojq.WithFunction("debug", 0, 0, cli.funcDebug), + gojq.WithFunction("stderr", 0, 0, cli.funcStderr), gojq.WithInputIter(iter), ) if err != nil { @@ -299,41 +307,26 @@ func (cli *cli) process(iter inputIter, code *gojq.Code) error { } } -func (cli *cli) printValues(v gojq.Iter) error { +func (cli *cli) printValues(iter gojq.Iter) error { m := cli.createMarshaler() for { - m, outStream := m, cli.outStream - x, ok := v.Next() + v, ok := iter.Next() if !ok { break } - switch v := x.(type) { - case error: - return v - case [2]interface{}: - if s, ok := v[0].(string); ok { - outStream = cli.errStream - compact := cli.outputCompact - cli.outputCompact = true - m = cli.createMarshaler() - cli.outputCompact = compact - if s == "STDERR:" { - x = v[1] - } else { - x = []interface{}{v[0], v[1]} - } - } + if err, ok := v.(error); ok { + return err } if cli.outputYAMLSeparator { - outStream.Write([]byte("---\n")) + cli.outStream.Write([]byte("---\n")) } else { cli.outputYAMLSeparator = cli.outputYAML } - if err := m.marshal(x, outStream); err != nil { + if err := m.marshal(v, cli.outStream); err != nil { return err } if cli.exitCodeError != nil { - if x == nil || x == false { + if v == nil || v == false { cli.exitCodeError = &exitCodeError{exitCodeFalsyErr} } else { cli.exitCodeError = &exitCodeError{exitCodeOK} @@ -341,9 +334,9 @@ func (cli *cli) printValues(v gojq.Iter) error { } if !cli.outputJoin && !cli.outputYAML { if cli.outputNul { - outStream.Write([]byte{'\x00'}) + cli.outStream.Write([]byte{'\x00'}) } else { - outStream.Write([]byte{'\n'}) + cli.outStream.Write([]byte{'\n'}) } } } @@ -369,6 +362,17 @@ func (cli *cli) createMarshaler() marshaler { return f } +func (cli *cli) funcDebug(v interface{}, _ []interface{}) interface{} { + newEncoder(false, 0).marshal([]interface{}{"DEBUG:", v}, cli.errStream) + cli.errStream.Write([]byte{'\n'}) + return v +} + +func (cli *cli) funcStderr(v interface{}, _ []interface{}) interface{} { + newEncoder(false, 0).marshal(v, cli.errStream) + return v +} + func (cli *cli) printError(err error) { if er, ok := err.(interface{ IsEmptyError() bool }); !ok || !er.IsEmptyError() { fmt.Fprintf(cli.errStream, "%s: %s\n", name, err) diff --git a/cli/cli_test.go b/cli/cli_test.go index 61f7c231..65d09d58 100644 --- a/cli/cli_test.go +++ b/cli/cli_test.go @@ -110,7 +110,7 @@ func TestCliRun(t *testing.T) { if got, expected := errorReplacer.Replace(errStr), strings.TrimSpace(tc.Error); !strings.Contains(got, expected) { t.Error("standard error output:\n" + cmp.Diff(expected, got)) } - if !strings.HasSuffix(errStr, "\n") { + if !strings.HasSuffix(errStr, "\n") && !strings.Contains(tc.Name, "stderr") { t.Error(`standard error output should end with "\n"`) } if strings.HasSuffix(errStr, "\n\n") { diff --git a/cli/test.yaml b/cli/test.yaml index 10d5f9c4..59dec0c9 100644 --- a/cli/test.yaml +++ b/cli/test.yaml @@ -3186,7 +3186,7 @@ - name: infinite function args: - -c - - '[infinite, -infinite] | (.[], .[]+.[], .[]-.[], .[]*.[], .[]/.[], .[]%.[], 1/.[], -1/.[], .[]-1e300, .[]+1e300) | [., . == infinite, . == -infinite, contains(infinite)]' + - '[infinite, -infinite] | (.[], .[]+.[], .[]-.[], .[]*.[], .[]/.[], .[]%.[]%256, 1/.[], -1/.[], .[]-1e300, .[]+1e300) | [., . == infinite, . == -infinite, contains(infinite)]' input: 'null' expected: | [1.7976931348623157e+308,true,false,true] @@ -3208,8 +3208,8 @@ [null,false,false,false] [null,false,false,false] [0,false,false,false] - [0,false,false,false] - [0,false,false,false] + [-1,false,false,false] + [255,false,false,false] [0,false,false,false] [0,false,false,false] [-0,false,false,false] @@ -4025,14 +4025,15 @@ } 2 "DEBUG:" - error: | - [{}] - {} - [1,{"a":2}] - 1 - {"a":2} - 2 - "DEBUG:" + error: '[{}]{}[1,{"a":2}]1{"a":2}2"DEBUG:"' + +- name: debug and stderr in builtins + args: + - 'builtins[] | select(test("debug|stderr"))' + input: 'null' + expected: | + "debug/0" + "stderr/0" - name: halt function args: @@ -4129,12 +4130,12 @@ expected: | true -- name: env in builtin +- name: env in builtins args: - - 'builtins | contains(["env/0"])' + - 'builtins[] | select(test("env"))' input: 'null' expected: | - true + "env/0" - name: $ENV variable args: @@ -6156,6 +6157,37 @@ input: 'null' error: 'open testdata/6.json:' +- name: $ARGS variable + args: + - --arg + - 'a' + - '1' + - --argjson + - 'b' + - '1' + - --slurpfile + - 'c' + - 'testdata/1.json' + - --rawfile + - 'd' + - 'testdata/1.json' + - '$ARGS | ., .named.a' + input: 'null' + expected: | + { + "named": { + "a": "1", + "b": 1, + "c": [ + { + "foo": 10 + } + ], + "d": "{\"foo\":10}\n" + } + } + "1" + - name: exit status option args: - -e diff --git a/code.go b/code.go index 963350b7..e44a78f3 100644 --- a/code.go +++ b/code.go @@ -35,7 +35,6 @@ const ( opexpend oppathbegin oppathend - opdebug ) func (op opcode) String() string { @@ -94,8 +93,6 @@ func (op opcode) String() string { return "pathbegin" case oppathend: return "pathend" - case opdebug: - return "debug" default: panic(op) } diff --git a/compiler.go b/compiler.go index 9d785c3d..5d813280 100644 --- a/compiler.go +++ b/compiler.go @@ -36,15 +36,15 @@ type Code struct { // // It is safe to call this method of a *Code in multiple goroutines. func (c *Code) Run(v interface{}, values ...interface{}) Iter { - return c.RunWithContext(nil, v, values...) + return c.RunWithContext(context.Background(), v, values...) } // RunWithContext runs the code with context. func (c *Code) RunWithContext(ctx context.Context, v interface{}, values ...interface{}) Iter { if len(values) > len(c.variables) { - return unitIterator(&tooManyVariableValuesError{}) + return NewIter(&tooManyVariableValuesError{}) } else if len(values) < len(c.variables) { - return unitIterator(&expectedVariableError{c.variables[len(values)]}) + return NewIter(&expectedVariableError{c.variables[len(values)]}) } for i, v := range values { values[i] = normalizeNumbers(v) @@ -60,8 +60,7 @@ func (c *Code) RunWithContext(ctx context.Context, v interface{}, values ...inte // LoadInitModules() ([]*Query, error) // LoadJSON(string) (interface{}, error) // LoadJSONWithMeta(string, map[string]interface{}) (interface{}, error) -type ModuleLoader interface { -} +type ModuleLoader interface{} type codeinfo struct { name string @@ -438,7 +437,7 @@ func (c *compiler) compileAlt(l, r *Query) error { return c.compileQuery(r) } -func (c *compiler) compileQueryUpdate(l, r *Query, op Operator) (err error) { +func (c *compiler) compileQueryUpdate(l, r *Query, op Operator) error { switch op { case OpAssign: // .foo.bar = f => setpath(["foo", "bar"]; f) @@ -736,7 +735,7 @@ func (c *compiler) compileLabel(e *Label) error { return c.compileQuery(e.Body) } -func (c *compiler) compileTerm(e *Term) (err error) { +func (c *compiler) compileTerm(e *Term) error { if len(e.SuffixList) > 0 { s := e.SuffixList[len(e.SuffixList)-1] t := *e // clone without changing e @@ -885,12 +884,6 @@ func (c *compiler) compileFunc(e *Func) error { } c.codes[len(c.codes)-1] = &code{op: oppathend} return nil - case "debug": - c.append(&code{op: opdebug, v: "DEBUG:"}) - return nil - case "stderr": - c.append(&code{op: opdebug, v: "STDERR:"}) - return nil case "builtins": return c.compileCallInternal( [3]interface{}{c.funcBuiltins, 0, e.Name}, @@ -920,12 +913,18 @@ func (c *compiler) compileFunc(e *Func) error { } } if fn, ok := c.customFuncs[e.Name]; ok && fn.accept(len(e.Args)) { - return c.compileCallInternal( + if err := c.compileCallInternal( [3]interface{}{fn.callback, len(e.Args), e.Name}, e.Args, nil, false, - ) + ); err != nil { + return err + } + if fn.iter { + c.append(&code{op: opeach}) + } + return nil } return &funcNotFoundError{e} } diff --git a/compiler_test.go b/compiler_test.go index da6bee55..870f0b26 100644 --- a/compiler_test.go +++ b/compiler_test.go @@ -150,7 +150,7 @@ func TestCodeCompile_OptimizeTailRec(t *testing.T) { t.Errorf("expected: %v, got: %v", expected, got) } iter := code.Run(nil) - var n int + n := 0 for { got, ok := iter.Next() if !ok { @@ -161,6 +161,9 @@ func TestCodeCompile_OptimizeTailRec(t *testing.T) { } n++ } + if expected := 10; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } } func TestCodeCompile_OptimizeJumps(t *testing.T) { @@ -207,7 +210,7 @@ func TestCodeRun_Race(t *testing.T) { go func() { defer wg.Done() iter := code.Run(nil) - var n int + n := 0 for { got, ok := iter.Next() if !ok { @@ -218,6 +221,9 @@ func TestCodeRun_Race(t *testing.T) { } n++ } + if expected := 10; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } }() } wg.Wait() @@ -235,7 +241,7 @@ func BenchmarkCompile(b *testing.B) { for i := 0; i < b.N; i++ { _, err := gojq.Compile( query, - gojq.WithInputIter(newTestInputIter(nil)), + gojq.WithInputIter(gojq.NewIter()), ) if err != nil { b.Fatal(err) diff --git a/debug.go b/debug.go index 806552fa..2e78846b 100644 --- a/debug.go +++ b/debug.go @@ -176,6 +176,9 @@ func debugOperand(c *code) string { } func debugJSON(v interface{}) string { + if _, ok := v.(Iter); ok { + return fmt.Sprintf("gojq.Iter(%#v)", v) + } var sb strings.Builder json.NewEncoder(&sb).Encode(v) return strings.TrimSpace(sb.String()) diff --git a/error.go b/error.go index 0dbdf5a8..f7670dbc 100644 --- a/error.go +++ b/error.go @@ -72,8 +72,7 @@ func (err *expectedStartEndError) Error() string { return `expected "start" and "end" for slicing but got: ` + typeErrorPreview(err.v) } -type inputNotAllowedError struct { -} +type inputNotAllowedError struct{} func (*inputNotAllowedError) Error() string { return "input(s)/0 is not allowed" @@ -297,6 +296,9 @@ func (err *jsonParseError) Error() string { } func typeErrorPreview(v interface{}) string { + if _, ok := v.(Iter); ok { + return "gojq.Iter" + } p := preview(v) if p != "" { p = " (" + p + ")" diff --git a/execute.go b/execute.go index a63388d9..6e7442a0 100644 --- a/execute.go +++ b/execute.go @@ -1,6 +1,7 @@ package gojq import ( + "context" "fmt" "reflect" "sort" @@ -20,7 +21,7 @@ func (env *env) execute(bc *Code, v interface{}, vars ...interface{}) Iter { func (env *env) Next() (interface{}, bool) { var err error pc, callpc, index := env.pc, len(env.codes)-1, -1 - backtrack, hasCtx := env.backtrack, env.ctx != nil + backtrack, hasCtx := env.backtrack, env.ctx != context.Background() defer func() { env.pc, env.backtrack = pc, true }() loop: for ; pc < len(env.codes); pc++ { @@ -222,7 +223,8 @@ loop: case [][2]interface{}: xs = v case []interface{}: - if !env.paths.empty() && (env.expdepth == 0 && !reflect.DeepEqual(v, env.paths.top().([2]interface{})[1])) { + if !env.paths.empty() && env.expdepth == 0 && + !reflect.DeepEqual(v, env.paths.top().([2]interface{})[1]) { err = &invalidPathIterError{v} break loop } @@ -234,7 +236,8 @@ loop: xs[i] = [2]interface{}{i, v} } case map[string]interface{}: - if !env.paths.empty() && (env.expdepth == 0 && !reflect.DeepEqual(v, env.paths.top().([2]interface{})[1])) { + if !env.paths.empty() && env.expdepth == 0 && + !reflect.DeepEqual(v, env.paths.top().([2]interface{})[1]) { err = &invalidPathIterError{v} break loop } @@ -250,6 +253,23 @@ loop: sort.Slice(xs, func(i, j int) bool { return xs[i][0].(string) < xs[j][0].(string) }) + case Iter: + if !env.paths.empty() && env.expdepth == 0 { + err = &invalidPathIterError{v} + break loop + } + if w, ok := v.Next(); ok { + env.push(v) + env.pushfork(code.op, pc) + env.pop() + if e, ok := w.(error); ok { + err = e + break loop + } + env.push(w) + continue + } + break loop default: err = &iteratorError{v} break loop @@ -260,10 +280,8 @@ loop: env.pop() } env.push(xs[0][1]) - if !env.paths.empty() { - if env.expdepth == 0 { - env.paths.push(xs[0]) - } + if !env.paths.empty() && env.expdepth == 0 { + env.paths.push(xs[0]) } case opexpbegin: env.expdepth++ @@ -289,11 +307,6 @@ loop: err = &invalidPathError{x} break loop } - case opdebug: - if !backtrack { - return [2]interface{}{code.v, env.stack.top()}, true - } - backtrack = false default: panic(code.op) } diff --git a/func.go b/func.go index dced0ff8..1e7bc6e0 100644 --- a/func.go +++ b/func.go @@ -29,6 +29,7 @@ const ( type function struct { argcount int + iter bool callback func(interface{}, []interface{}) interface{} } @@ -42,8 +43,6 @@ func init() { internalFuncs = map[string]function{ "empty": argFunc0(nil), "path": argFunc1(nil), - "debug": argFunc0(nil), - "stderr": argFunc0(nil), "env": argFunc0(nil), "builtins": argFunc0(nil), "input": argFunc0(nil), @@ -60,7 +59,7 @@ func init() { "contains": argFunc1(funcContains), "explode": argFunc0(funcExplode), "implode": argFunc0(funcImplode), - "split": {argcount1 | argcount2, funcSplit}, + "split": {argcount1 | argcount2, false, funcSplit}, "tojson": argFunc0(funcToJSON), "fromjson": argFunc0(funcFromJSON), "format": argFunc1(funcFormat), @@ -172,16 +171,16 @@ func init() { "strptime": argFunc1(funcStrptime), "now": argFunc0(funcNow), "_match": argFunc3(funcMatch), - "error": {argcount0 | argcount1, funcError}, + "error": {argcount0 | argcount1, false, funcError}, "halt": argFunc0(funcHalt), - "halt_error": {argcount0 | argcount1, funcHaltError}, + "halt_error": {argcount0 | argcount1, false, funcHaltError}, "_type_error": argFunc1(internalfuncTypeError), } } func argFunc0(fn func(interface{}) interface{}) function { return function{ - argcount0, func(v interface{}, _ []interface{}) interface{} { + argcount0, false, func(v interface{}, _ []interface{}) interface{} { return fn(v) }, } @@ -189,7 +188,7 @@ func argFunc0(fn func(interface{}) interface{}) function { func argFunc1(fn func(interface{}, interface{}) interface{}) function { return function{ - argcount1, func(v interface{}, args []interface{}) interface{} { + argcount1, false, func(v interface{}, args []interface{}) interface{} { return fn(v, args[0]) }, } @@ -197,7 +196,7 @@ func argFunc1(fn func(interface{}, interface{}) interface{}) function { func argFunc2(fn func(interface{}, interface{}, interface{}) interface{}) function { return function{ - argcount2, func(v interface{}, args []interface{}) interface{} { + argcount2, false, func(v interface{}, args []interface{}) interface{} { return fn(v, args[0], args[1]) }, } @@ -205,7 +204,7 @@ func argFunc2(fn func(interface{}, interface{}, interface{}) interface{}) functi func argFunc3(fn func(interface{}, interface{}, interface{}, interface{}) interface{}) function { return function{ - argcount3, func(v interface{}, args []interface{}) interface{} { + argcount3, false, func(v interface{}, args []interface{}) interface{} { return fn(v, args[0], args[1], args[2]) }, } @@ -1591,13 +1590,7 @@ func toInt(x interface{}) (int, bool) { case int: return x, true case float64: - if minInt <= x && x <= maxInt { - return int(x), true - } - if x > 0 { - return maxInt, true - } - return minInt, true + return floatToInt(x), true case *big.Int: if x.IsInt64() { if i := x.Int64(); minInt <= i && i <= maxInt { @@ -1613,6 +1606,16 @@ func toInt(x interface{}) (int, bool) { } } +func floatToInt(x float64) int { + if minInt <= x && x <= maxInt { + return int(x) + } + if x > 0 { + return maxInt + } + return minInt +} + func toFloat(x interface{}) (float64, bool) { switch x := x.(type) { case int: diff --git a/go.dev.mod b/go.dev.mod index 23cb84c6..950cbff7 100644 --- a/go.dev.mod +++ b/go.dev.mod @@ -3,6 +3,6 @@ module github.com/itchyny/gojq go 1.14 require ( - github.com/itchyny/astgen-go v0.0.0-20210222032259-bf31276dfbe1 // indirect - github.com/itchyny/timefmt-go v0.1.2 + github.com/itchyny/astgen-go v0.0.0-20210521111535-120f8643907b // indirect + github.com/itchyny/timefmt-go v0.1.3 ) diff --git a/go.dev.sum b/go.dev.sum index b78c94a4..8e7820bc 100644 --- a/go.dev.sum +++ b/go.dev.sum @@ -1,4 +1,4 @@ -github.com/itchyny/astgen-go v0.0.0-20210222032259-bf31276dfbe1 h1:z3NR0toPa8yJsPd21cmxnttYX+cDzBQSxZ569ICn49I= -github.com/itchyny/astgen-go v0.0.0-20210222032259-bf31276dfbe1/go.mod h1:296z3W7Xsrp2mlIY88ruDKscuvrkL6zXCNRtaYVshzw= -github.com/itchyny/timefmt-go v0.1.2 h1:q0Xa4P5it6K6D7ISsbLAMwx1PnWlixDcJL6/sFs93Hs= -github.com/itchyny/timefmt-go v0.1.2/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/itchyny/astgen-go v0.0.0-20210521111535-120f8643907b h1:BrOi0AhM/uyco/Rv3+Xf0rDiOfT6vWw4vGUd/Bkm0qI= +github.com/itchyny/astgen-go v0.0.0-20210521111535-120f8643907b/go.mod h1:296z3W7Xsrp2mlIY88ruDKscuvrkL6zXCNRtaYVshzw= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= diff --git a/go.mod b/go.mod index 14405f30..66f87777 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.14 require ( github.com/google/go-cmp v0.5.4 github.com/itchyny/go-flags v1.5.0 - github.com/itchyny/timefmt-go v0.1.2 - github.com/mattn/go-isatty v0.0.12 + github.com/itchyny/timefmt-go v0.1.3 + github.com/mattn/go-isatty v0.0.13 github.com/mattn/go-runewidth v0.0.9 - golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b // indirect + golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/go.sum b/go.sum index 937095d0..4f9bafd4 100644 --- a/go.sum +++ b/go.sum @@ -2,16 +2,16 @@ github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/itchyny/go-flags v1.5.0 h1:Z5q2ist2sfDjDlExVPBrMqlsEDxDR2h4zuOElB0OEYI= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= -github.com/itchyny/timefmt-go v0.1.2 h1:q0Xa4P5it6K6D7ISsbLAMwx1PnWlixDcJL6/sFs93Hs= -github.com/itchyny/timefmt-go v0.1.2/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= +github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b h1:kHlr0tATeLRMEiZJu5CknOw/E8V6h69sXXQFGoPtjcc= -golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk= +golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= diff --git a/iter.go b/iter.go index 4f724417..0cee25ba 100644 --- a/iter.go +++ b/iter.go @@ -5,19 +5,45 @@ type Iter interface { Next() (interface{}, bool) } -func unitIterator(v interface{}) Iter { - return &unitIter{v: v} +// NewIter creates a new Iter from values. +func NewIter(values ...interface{}) Iter { + switch len(values) { + case 0: + return emptyIter{} + case 1: + return &unitIter{value: values[0]} + default: + iter := sliceIter(values) + return &iter + } +} + +type emptyIter struct{} + +func (emptyIter) Next() (interface{}, bool) { + return nil, false } type unitIter struct { - v interface{} - done bool + value interface{} + done bool } -func (c *unitIter) Next() (interface{}, bool) { - if !c.done { - c.done = true - return c.v, true +func (iter *unitIter) Next() (interface{}, bool) { + if iter.done { + return nil, false } - return nil, false + iter.done = true + return iter.value, true +} + +type sliceIter []interface{} + +func (iter *sliceIter) Next() (interface{}, bool) { + if len(*iter) == 0 { + return nil, false + } + value := (*iter)[0] + *iter = (*iter)[1:] + return value, true } diff --git a/operator.go b/operator.go index a9f61c8a..0135922a 100644 --- a/operator.go +++ b/operator.go @@ -478,13 +478,11 @@ func funcOpMod(_, l, r interface{}) interface{} { return l % r }, func(l, r float64) interface{} { - if int(r) == 0 { - if r < -1.0 || 1.0 < r { // int(math.Inf(1)) == 0 on some architectures - return int(l) % minInt - } + ri := floatToInt(r) + if ri == 0 { return &zeroModuloError{l, r} } - return int(l) % int(r) + return floatToInt(l) % ri }, func(l, r *big.Int) interface{} { if r.Sign() == 0 { diff --git a/option.go b/option.go index 874c94da..33f0abd3 100644 --- a/option.go +++ b/option.go @@ -39,6 +39,25 @@ func WithVariables(variables []string) CompilerOption { // for its argument, or call another built-in function, then use LoadInitModules // of the module loader. func WithFunction(name string, minarity, maxarity int, + f func(interface{}, []interface{}) interface{}) CompilerOption { + return withFunction(name, minarity, maxarity, false, f) +} + +// WithIterFunction is a compiler option for adding a custom iterator function. +// This is like the WithFunction option, but you can add a function which +// returns an Iter to emit multiple values. You cannot define both iterator and +// non-iterator functions of the same name (with possibly different arities). +// See also NewIter, which can be used to convert values or an error to an Iter. +func WithIterFunction(name string, minarity, maxarity int, + f func(interface{}, []interface{}) Iter) CompilerOption { + return withFunction(name, minarity, maxarity, true, + func(v interface{}, args []interface{}) interface{} { + return f(v, args) + }, + ) +} + +func withFunction(name string, minarity, maxarity int, iter bool, f func(interface{}, []interface{}) interface{}) CompilerOption { if !(0 <= minarity && minarity <= maxarity && maxarity <= 30) { panic(fmt.Sprintf("invalid arity for %q: %d, %d", name, minarity, maxarity)) @@ -49,8 +68,11 @@ func WithFunction(name string, minarity, maxarity int, c.customFuncs = make(map[string]function) } if fn, ok := c.customFuncs[name]; ok { + if fn.iter != iter { + panic(fmt.Sprintf("cannot define both iterator and non-iterator functions for %q", name)) + } c.customFuncs[name] = function{ - argcount | fn.argcount, + argcount | fn.argcount, iter, func(x interface{}, xs []interface{}) interface{} { if argcount&(1<= iter.max { + return nil, false + } + v := iter.value + iter.value++ + return v, true +} + +func ExampleWithIterFunction() { + query, err := gojq.Parse("f(3; 7)") + if err != nil { + log.Fatalln(err) + } + code, err := gojq.Compile( + query, + gojq.WithIterFunction("f", 2, 2, func(_ interface{}, xs []interface{}) gojq.Iter { + if x, ok := xs[0].(int); ok { + if y, ok := xs[1].(int); ok { + return &rangeIter{x, y} + } + } + return gojq.NewIter(fmt.Errorf("f cannot be applied to: %v", xs)) + }), + ) + if err != nil { + log.Fatalln(err) + } + iter := code.Run(nil) + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + log.Fatalln(err) + } + fmt.Printf("%#v\n", v) + } + + // Output: + // 3 + // 4 + // 5 + // 6 +} diff --git a/option_test.go b/option_test.go index a1518285..b7cc35f5 100644 --- a/option_test.go +++ b/option_test.go @@ -2,10 +2,10 @@ package gojq_test import ( "encoding/json" + "errors" "fmt" "math/big" "reflect" - "strings" "testing" "github.com/itchyny/gojq" @@ -33,16 +33,17 @@ func TestWithModuleLoaderError(t *testing.T) { t.Fatal(err) } iter := code.Run("m") - for { - v, ok := iter.Next() - if !ok { - break - } - err := v.(error) - if got, expected := err.Error(), `cannot load module: "m"`; got != expected { - t.Errorf("expected: %v, got: %v", expected, got) - } - break + v, ok := iter.Next() + if !ok { + t.Fatal("should emit an error but got no output") + } + err, expected := v.(error), `cannot load module: "m"` + if got := err.Error(); got != expected { + t.Errorf("expected: %v, got: %v", expected, got) + } + v, ok = iter.Next() + if ok { + t.Errorf("should not emit a value but got: %v", v) } } @@ -239,16 +240,17 @@ func TestWithVariablesError1(t *testing.T) { t.Fatal(err) } iter := code.Run(nil) - for { - v, ok := iter.Next() - if !ok { - break - } - if err, ok := v.(error); ok { - if got, expected := err.Error(), "variable defined but not bound: $x"; got != expected { - t.Errorf("expected: %v, got: %v", expected, got) - } - } + v, ok := iter.Next() + if !ok { + t.Fatal("should emit an error but got no output") + } + err, expected := v.(error), "variable defined but not bound: $x" + if got := err.Error(); got != expected { + t.Errorf("expected: %v, got: %v", expected, got) + } + v, ok = iter.Next() + if ok { + t.Errorf("should not emit a value but got: %v", v) } } @@ -265,16 +267,17 @@ func TestWithVariablesError2(t *testing.T) { t.Fatal(err) } iter := code.Run(nil, 1, 2) - for { - v, ok := iter.Next() - if !ok { - break - } - if err, ok := v.(error); ok { - if got, expected := err.Error(), "too many variable values provided"; got != expected { - t.Errorf("expected: %v, got: %v", expected, got) - } - } + v, ok := iter.Next() + if !ok { + t.Fatal("should emit an error but got no output") + } + err, expected := v.(error), "too many variable values provided" + if got := err.Error(); got != expected { + t.Errorf("expected: %v, got: %v", expected, got) + } + v, ok = iter.Next() + if ok { + t.Errorf("should not emit a value but got: %v", v) } } @@ -315,6 +318,9 @@ func TestWithFunction(t *testing.T) { } n++ } + if expected := 5; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } query, err = gojq.Parse( `("f/0", "f/1", "g/0", "g/1") as $f | builtins | any(. == $f)`, ) @@ -337,6 +343,9 @@ func TestWithFunction(t *testing.T) { } n++ } + if expected := 4; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } } func TestWithFunctionDuplicateName(t *testing.T) { @@ -392,6 +401,9 @@ func TestWithFunctionDuplicateName(t *testing.T) { } n++ } + if expected := 5; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } query, err = gojq.Parse( `("f/0", "f/1", "f/2", "f/3") as $f | builtins | any(. == $f)`, ) @@ -414,6 +426,9 @@ func TestWithFunctionDuplicateName(t *testing.T) { } n++ } + if expected := 4; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } } func TestWithFunctionMultipleArities(t *testing.T) { @@ -465,6 +480,9 @@ func TestWithFunctionMultipleArities(t *testing.T) { } n++ } + if expected := 5; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } query, err = gojq.Parse( `("f/0", "f/1", "f/2", "f/3", "f/4", "f/5") as $f | builtins | any(. == $f)`, ) @@ -487,6 +505,9 @@ func TestWithFunctionMultipleArities(t *testing.T) { } n++ } + if expected := 6; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } } type valueError struct { @@ -527,6 +548,195 @@ func TestWithFunctionValueError(t *testing.T) { } } +func TestWithFunctionCompileArgsError(t *testing.T) { + query, err := gojq.Parse("f(g)") + if err != nil { + t.Fatal(err) + } + _, err = gojq.Compile(query, + gojq.WithFunction("f", 0, 1, func(interface{}, []interface{}) interface{} { + return 0 + }), + ) + if got, expected := err.Error(), "function not defined: g/0"; got != expected { + t.Errorf("expected: %v, got: %v", expected, got) + } +} + +func TestWithFunctionArityError(t *testing.T) { + query, err := gojq.Parse("f") + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ min, max int }{{3, 2}, {-1, 3}, {0, 31}, {-1, 31}} { + func() { + defer func() { + expected := fmt.Sprintf(`invalid arity for "f": %d, %d`, tc.min, tc.max) + if got := recover(); got != expected { + t.Errorf("expected: %v, got: %v", expected, got) + } + }() + t.Fatal(gojq.Compile(query, + gojq.WithFunction("f", tc.min, tc.max, func(interface{}, []interface{}) interface{} { + return 0 + }), + )) + }() + } +} + +func TestWithIterFunction(t *testing.T) { + query, err := gojq.Parse("f * g(5; 10), h, 10") + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query, + gojq.WithIterFunction("f", 0, 0, func(interface{}, []interface{}) gojq.Iter { + return gojq.NewIter(1, 2, 3) + }), + gojq.WithIterFunction("g", 2, 2, func(_ interface{}, xs []interface{}) gojq.Iter { + if x, ok := xs[0].(int); ok { + if y, ok := xs[1].(int); ok { + return &rangeIter{x, y} + } + } + return gojq.NewIter(fmt.Errorf("g cannot be applied to: %v", xs)) + }), + gojq.WithIterFunction("h", 0, 0, func(interface{}, []interface{}) gojq.Iter { + return gojq.NewIter() + }), + ) + if err != nil { + t.Fatal(err) + } + iter := code.Run(nil) + n := 0 + for { + v, ok := iter.Next() + if !ok { + break + } + if expected := (1 + n%3) * (5 + n/3); v != expected { + t.Errorf("expected: %v, got: %v", expected, v) + } + n++ + } + if expected := 16; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } +} + +func TestWithIterFunctionError(t *testing.T) { + query, err := gojq.Parse("range(3) * (f, 0), f") + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query, + gojq.WithIterFunction("f", 0, 0, func(interface{}, []interface{}) gojq.Iter { + return gojq.NewIter(1, errors.New("error"), 3) + }), + ) + if err != nil { + t.Fatal(err) + } + iter := code.Run(nil) + n := 0 + for { + v, ok := iter.Next() + if !ok { + break + } + switch n { + case 0, 1, 2: + if expected := n; v != expected { + t.Errorf("expected: %v, got: %v", expected, v) + } + case 3, 5: + if expected := "error"; v.(error).Error() != expected { + t.Errorf("expected: %v, got: %v", expected, err) + } + default: + if expected := n - 3; v != expected { + t.Errorf("expected: %v, got: %v", expected, v) + } + } + n++ + } + if expected := 7; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } +} + +func TestWithIterFunctionPath(t *testing.T) { + query, err := gojq.Parse(".[f] = 1") + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query, + gojq.WithIterFunction("f", 0, 0, func(interface{}, []interface{}) gojq.Iter { + return gojq.NewIter(0, 1, 2) + }), + ) + if err != nil { + t.Fatal(err) + } + iter := code.Run(nil) + for { + v, ok := iter.Next() + if !ok { + break + } + if expected := []interface{}{1, 1, 1}; !reflect.DeepEqual(v, expected) { + t.Errorf("expected: %v, got: %v", expected, v) + } + } +} + +func TestWithIterFunctionPathError(t *testing.T) { + query, err := gojq.Parse("{x: 0} | (f|.x) = 1") + if err != nil { + t.Fatal(err) + } + code, err := gojq.Compile(query, + gojq.WithIterFunction("f", 0, 0, func(interface{}, []interface{}) gojq.Iter { + return gojq.NewIter(map[string]interface{}{"x": 0}) + }), + ) + if err != nil { + t.Fatal(err) + } + iter := code.Run(nil) + v, ok := iter.Next() + if !ok { + t.Fatal("should emit an error but got no output") + } + err, expected := v.(error), "invalid path on iterating against: gojq.Iter" + if got := err.Error(); got != expected { + t.Errorf("expected: %v, got: %v", expected, got) + } +} + +func TestWithIterFunctionDefineError(t *testing.T) { + query, err := gojq.Parse("f") + if err != nil { + t.Fatal(err) + } + defer func() { + expected := `cannot define both iterator and non-iterator functions for "f"` + if got := recover(); got != expected { + t.Errorf("expected: %v, got: %v", expected, got) + } + }() + t.Fatal(gojq.Compile(query, + gojq.WithFunction("f", 0, 0, func(interface{}, []interface{}) interface{} { + return 0 + }), + gojq.WithIterFunction("f", 0, 0, func(interface{}, []interface{}) gojq.Iter { + return gojq.NewIter() + }), + )) +} + type moduleLoader2 struct{} func (*moduleLoader2) LoadModule(name string) (*gojq.Query, error) { @@ -568,6 +778,9 @@ func TestWithFunctionWithModuleLoader(t *testing.T) { } n++ } + if expected := 5; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } } func TestWithInputIter(t *testing.T) { @@ -577,15 +790,13 @@ func TestWithInputIter(t *testing.T) { } code, err := gojq.Compile( query, - gojq.WithInputIter( - newTestInputIter(strings.NewReader("1 2 3 4 5")), - ), + gojq.WithInputIter(gojq.NewIter(0, 1, 2, 3, 4)), ) if err != nil { t.Fatal(err) } iter := code.Run(nil) - n := 1 + n := 0 for { v, ok := iter.Next() if !ok { @@ -595,11 +806,12 @@ func TestWithInputIter(t *testing.T) { if expected := "break"; err.Error() != expected { t.Errorf("expected: %v, got: %v", expected, err) } - break - } - if v != n { + } else if v != n { t.Errorf("expected: %v, got: %v", n, v) } n++ } + if expected := 10; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } } diff --git a/query.go b/query.go index 4e2ef811..53e29d70 100644 --- a/query.go +++ b/query.go @@ -23,14 +23,14 @@ type Query struct { // // It is safe to call this method of a *Query in multiple goroutines. func (e *Query) Run(v interface{}) Iter { - return e.RunWithContext(nil, v) + return e.RunWithContext(context.Background(), v) } // RunWithContext runs the query with context. func (e *Query) RunWithContext(ctx context.Context, v interface{}) Iter { code, err := Compile(e) if err != nil { - return unitIterator(err) + return NewIter(err) } return code.RunWithContext(ctx, v) } diff --git a/query_test.go b/query_test.go index f6797e6d..2489ae9f 100644 --- a/query_test.go +++ b/query_test.go @@ -88,6 +88,9 @@ func TestQueryRun_Errors(t *testing.T) { } n++ } + if expected := 5; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } } func TestQueryRun_ObjectError(t *testing.T) { @@ -158,6 +161,22 @@ func TestQueryRun_NumericTypes(t *testing.T) { } } +func TestQueryRun_Input(t *testing.T) { + query, err := gojq.Parse("input") + if err != nil { + t.Fatal(err) + } + iter := query.Run(nil) + v, ok := iter.Next() + if !ok { + t.Fatal("should emit an error but got no output") + } + err, expected := v.(error), "input(s)/0 is not allowed" + if got := err.Error(); got != expected { + t.Errorf("expected: %v, got: %v", expected, got) + } +} + func TestQueryRun_Race(t *testing.T) { query, err := gojq.Parse("range(10)") if err != nil { @@ -169,7 +188,7 @@ func TestQueryRun_Race(t *testing.T) { go func() { defer wg.Done() iter := query.Run(nil) - var n int + n := 0 for { got, ok := iter.Next() if !ok { @@ -180,6 +199,9 @@ func TestQueryRun_Race(t *testing.T) { } n++ } + if expected := 10; n != expected { + t.Errorf("expected: %v, got: %v", expected, n) + } }() } wg.Wait()