diff --git a/cmd/mackerel-plugin-jsonl/main.go b/cmd/mackerel-plugin-jsonl/main.go index c21aab7..1d3b166 100644 --- a/cmd/mackerel-plugin-jsonl/main.go +++ b/cmd/mackerel-plugin-jsonl/main.go @@ -28,6 +28,7 @@ type Opt struct { filterByte *[]byte ignoreByte *[]byte paths [][]string + flatPaths bool duration float64 } diff --git a/cmd/mackerel-plugin-jsonl/main_test.go b/cmd/mackerel-plugin-jsonl/main_test.go index f1c6008..e6d6a4c 100644 --- a/cmd/mackerel-plugin-jsonl/main_test.go +++ b/cmd/mackerel-plugin-jsonl/main_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/monitoring-forge/followparser" + "github.com/stretchr/testify/require" ) func generateJSONLFile(b testing.TB, dir, filename string, numLines int) error { @@ -90,7 +91,7 @@ func initParserForTest(b testing.TB, tmpDir string) (*followparser.Parser, *Opt) return fp, opt } -func internalBenchmarkParse(b *testing.B, numLines int, doOutput bool) { +func internalBenchmarkParse(b *testing.B, numLines int, doOutput, useEachKey bool) { tmpDir := b.TempDir() prefix := "json" logFileName := "json.log" @@ -107,44 +108,39 @@ func internalBenchmarkParse(b *testing.B, numLines int, doOutput bool) { for b.Loop() { b.StopTimer() err := resetFollowParserStateFile(b, tmpDir, logFileName, prefix) - if err != nil { - b.Fatalf("resetFollowParserStateFile failed: %v", err) - } + require.NoError(b, err, "resetFollowParserStateFile failed") b.StartTimer() fp, opt := initParserForTest(b, tmpDir) + if useEachKey { + opt.flatPaths = false + } parsed, err := fp.Parse( posFile, logFile, ) - if err != nil { - b.Fatalf("Parse failed: %v", err) - } + require.NoError(b, err, "Parse failed") if doOutput { output := opt.output() - if output == "" { - b.Fatalf("output is empty") - } + require.NotEmpty(b, output, "output is empty") } b.StopTimer() - if parsed == nil { - b.Fatalf("Parse returned nil parsed data") - } - if len(parsed) != 1 { - b.Fatalf("Parse returned unexpected number of parsed data: got %d, want 1", len(parsed)) - } - if parsed[0].Rows != numLines { - b.Fatalf("Parse returned unexpected number of rows: got %d, want %d", parsed[0].Rows, numLines) - } + require.NotNil(b, parsed, "parsed data is nil") + require.Equal(b, 1, len(parsed), "unexpected number of parsed data") + require.Equal(b, numLines, parsed[0].Rows, "unexpected number of rows in parsed data") b.StartTimer() } } // generate 100k JSONL file and parse benchmark func BenchmarkMainParse_jsonl(b *testing.B) { - internalBenchmarkParse(b, 100_000, false) + internalBenchmarkParse(b, 100_000, false, false) +} + +func BenchmarkMainParse_jsonl_eachkey(b *testing.B) { + internalBenchmarkParse(b, 100_000, false, true) } // generate 100k JSONL file and parse benchmark func BenchmarkMainParse_parse_and_output(b *testing.B) { - internalBenchmarkParse(b, 100_000, true) + internalBenchmarkParse(b, 100_000, true, false) } diff --git a/cmd/mackerel-plugin-jsonl/parser.go b/cmd/mackerel-plugin-jsonl/parser.go index 4f02bb5..a4223aa 100644 --- a/cmd/mackerel-plugin-jsonl/parser.go +++ b/cmd/mackerel-plugin-jsonl/parser.go @@ -2,11 +2,14 @@ package main import ( "bytes" + "errors" "log" "github.com/buger/jsonparser" ) +var errFlatPathsFound = errors.New("all flat JSON paths found") + func (opt *Opt) jsonParsed(idx int, value []byte, vt jsonparser.ValueType, err error) { if err != nil { log.Printf("error: %v", err) @@ -37,10 +40,45 @@ func (opt *Opt) Parse(b []byte) error { } } - jsonparser.EachKey(b, opt.jsonParsed, opt.paths...) + if opt.flatPaths { + opt.parseFlat(b) + } else { + jsonparser.EachKey(b, opt.jsonParsed, opt.paths...) + } return nil } +// parseFlat reads only root-level keys. ObjectEach decodes escaped key names, +// so comparisons here use the same names as paths passed to EachKey. +func (opt *Opt) parseFlat(b []byte) { + var found uint64 + remaining := len(opt.paths) + err := jsonparser.ObjectEach(b, func(key, value []byte, valueType jsonparser.ValueType, _ int) error { + for i, path := range opt.paths { + bit := uint64(1) << i + if found&bit != 0 || !bytes.Equal(key, []byte(path[0])) { + continue + } + found |= bit + remaining-- + opt.jsonParsed(i, value, valueType, nil) + } + if remaining == 0 { + return errFlatPathsFound + } + return nil + }) + if err != nil && err != errFlatPathsFound { //nolint:errorlint + // Keep EachKey's behavior for non-object and malformed input. Skip values + // already delivered by ObjectEach so aggregators never count them twice. + jsonparser.EachKey(b, func(i int, value []byte, valueType jsonparser.ValueType, err error) { + if i < 0 || found&(uint64(1)< 0 && len(p.aggregatorFunctions) <= 64 for _, af := range p.aggregatorFunctions { paths = append(paths, af.jsonKey) + if len(af.jsonKey) != 1 { + p.flatPaths = false + } } p.paths = paths }